initialization for a multi-chunk upload
333135
def init_chunked_upload
      headers = {
        'x-ton-content-type'   => content_type,
        'x-ton-content-length' => @file_size,
        'x-ton-expires'        => DEFAULT_EXPIRE,
        'content-length'       => 0,
        'content-type'         => content_type
      }
      resource = "#{DEFAULT_RESOURCE}#{@bucket}?resumable=true"
      TwitterAds::Request.new(
        @client, :post, resource, domain: DEFAULT_DOMAIN, headers: headers).perform
    end
1311037
private function initBufferFromNextChunk()
    {
        if ($this->chunkOffset === $this->numChunks - 1) {
            return false;
        }

        $this->bufferOffset = 0;
        $this->chunkOffset++;
        $this->chunksIterator->next();

        return $this->initBufferFromCurrentChunk();
    }
850426
async def _upload_chunks(
            cls, rfile: BootResourceFile, content: io.IOBase, chunk_size: int,
            progress_callback=None):
        """"""
        content.seek(0, io.SEEK_SET)
        upload_uri = urlparse(
            cls._handler.uri)._replace(path=rfile._data['upload_uri']).geturl()
        uploaded_size = 0

        insecure = cls._handler.session.insecure
        connector = aiohttp.TCPConnector(verify_ssl=(not insecure))
        session = aiohttp.ClientSession(connector=connector)

        async with session:
            while True:
                buf = content.read(chunk_size)
                length = len(buf)
                if length > 0:
                    uploaded_size += length
                    await cls._put_chunk(session, upload_uri, buf)
                    if progress_callback is not None:
                        progress_callback(uploaded_size / rfile.size)
                if length != chunk_size:
                    break
823416
public void setUploadChunkSize(int chunkSize) {
    Preconditions.checkArgument(chunkSize > 0, "Upload chunk size must be great than 0.");
    Preconditions.checkArgument(
        chunkSize % MediaHttpUploader.MINIMUM_CHUNK_SIZE == 0,
        "Upload chunk size must be a multiple of MediaHttpUploader.MINIMUM_CHUNK_SIZE");
    if ((chunkSize > GCS_UPLOAD_GRANULARITY) && (chunkSize % GCS_UPLOAD_GRANULARITY != 0)) {
      logger.atWarning().log(
          "Upload chunk size should be a multiple of %s for best performance, got %s",
          GCS_UPLOAD_GRANULARITY, chunkSize);
    }
    uploadChunkSize = chunkSize;
  }
442694
function InitiateMultipartUpload(fileUpload, awsKey) {
            var request = {
                method: 'POST',
                path: '?uploads',
                step: 'initiate',
                x_amz_headers: fileUpload.xAmzHeadersAtInitiate,
                not_signed_headers: fileUpload.notSignedHeadersAtInitiate,
                response_match: '<UploadId>(.+)<\/UploadId>'
            };

            CancelableS3AWSRequest.call(this, fileUpload, request);
            this.awsKey = awsKey;
        }
330330
public void asyncPut(final byte[] data, final String key, final String token, StringMap params,
                         String mime, boolean checkCrc, UpCompletionHandler handler) throws IOException {
        checkArgs(key, data, null, token);
        if (mime == null) {
            mime = Client.DefaultMime;
        }
        params = filterParam(params);
        new FormUploader(client, token, key, data, params, mime, checkCrc, configuration).asyncUpload(handler);
    }
171184
private void initialize(Chunk chunk)
    {
        final String mName = "initialize";

        if(logger.isLoggable(Level.FINER)) 
            logger.entering(className, mName);

        try
        {
            if (chunk.getRetryLimit() != null){
                _retryLimit = Integer.parseInt(chunk.getRetryLimit());
                if (_retryLimit < 0) {
                    throw new IllegalArgumentException("The retry-limit attribute on a chunk cannot be a negative value");
                }

            }
        }
        catch (NumberFormatException nfe)
        {
            throw new RuntimeException("NumberFormatException reading " + RETRY_COUNT, nfe);
        }

        // Read the include/exclude exceptions.
        _retryIncludeExceptions = new HashSet<String>();
        _retryExcludeExceptions = new HashSet<String>();
        _retryNoRBIncludeExceptions = new HashSet<String>();
        _retryNoRBExcludeExceptions = new HashSet<String>();

        if (chunk.getRetryableExceptionClasses() != null) {
            if (chunk.getRetryableExceptionClasses().getIncludeList() != null) {
                List<ExceptionClassFilter.Include> includes = chunk.getRetryableExceptionClasses().getIncludeList();
                for (ExceptionClassFilter.Include include : includes) {
                    _retryIncludeExceptions.add(include.getClazz().trim());
                    logger.finer("RETRYHANDLE: include: " + include.getClazz().trim());
                }

                if (_retryIncludeExceptions.size() == 0) {
                    logger.finer("RETRYHANDLE: include element not present");

                }
            }
            if (chunk.getRetryableExceptionClasses().getExcludeList() != null) {
                List<ExceptionClassFilter.Exclude> excludes = chunk.getRetryableExceptionClasses().getExcludeList();
                for (ExceptionClassFilter.Exclude exclude : excludes) {
                    _retryExcludeExceptions.add(exclude.getClazz().trim());
                    logger.finer("SKIPHANDLE: exclude: " + exclude.getClazz().trim());
                }

                if (_retryExcludeExceptions.size() == 0) {
                    logger.finer("SKIPHANDLE: exclude element not present");

                }
            }
        }

        if (chunk.getNoRollbackExceptionClasses() != null) {
            if (chunk.getNoRollbackExceptionClasses().getIncludeList() != null) {
                List<ExceptionClassFilter.Include> includes = chunk.getNoRollbackExceptionClasses().getIncludeList();
                for (ExceptionClassFilter.Include include : includes) {
                    _retryNoRBIncludeExceptions.add(include.getClazz().trim());
                    logger.finer("RETRYHANDLE: include: " + include.getClazz().trim());
                }

                if (_retryNoRBIncludeExceptions.size() == 0) {
                    logger.finer("RETRYHANDLE: include element not present");

                }
            }
            if (chunk.getNoRollbackExceptionClasses().getExcludeList() != null) {
                List<ExceptionClassFilter.Exclude> excludes = chunk.getNoRollbackExceptionClasses().getExcludeList();
                for (ExceptionClassFilter.Exclude exclude : excludes) {
                    _retryNoRBExcludeExceptions.add(exclude.getClazz().trim());
                    logger.finer("SKIPHANDLE: exclude: " + exclude.getClazz().trim());
                }

                if (_retryNoRBExcludeExceptions.size() == 0) {
                    logger.finer("SKIPHANDLE: exclude element not present");

                }
            }
        }
        
        // Create the ExceptionMatcher Object
        excMatcher = new ExceptionMatcher(_retryIncludeExceptions, _retryExcludeExceptions);
        noRBexcMatcher = new ExceptionMatcher(_retryNoRBIncludeExceptions, _retryNoRBExcludeExceptions);


        if (logger.isLoggable(Level.FINE)) {
            logger.logp(Level.FINE, className, mName, "added include exception " + _retryIncludeExceptions 
                    + "; added exclude exception " + _retryExcludeExceptions);
            logger.logp(Level.FINE, className, mName, "added include no rollback exception " + _retryNoRBIncludeExceptions
                    + "; added exclude no rollback exception " + _retryNoRBExcludeExceptions);
        }
            
        if(logger.isLoggable(Level.FINER)) {
          logger.exiting(className, mName, this.toString());
        }
      }
1025932
async function writeChunk(){
        if (chunks.length > 0){
            // get first element
            const l = chunks.shift();

            // increment size counter
            currentUploadSize += l.length;

            // write first element to file
            try{
                await _virtualTerminal.executeCommand('__nmtwrite("' + l + '")');
            }catch(e){
                _logger.debug(e);
                throw new Error('cannot write chunk to remote file');
            }

            // run progress callback
            progressCb.apply(progressCb, [currentUploadSize, absoluteFilesize]);

            // write next
            await writeChunk();

        }else{
            // ensure that the progress callback is called, even for empty files
            progressCb.apply(progressCb, [currentUploadSize, absoluteFilesize]);

            // send file close command
            try{
                await _virtualTerminal.executeCommand(_luaCommandBuilder.command.fileCloseFlush);
            }catch(e){
                _logger.debug(e);
                throw new Error('cannot flush/close remote file');
            }
        }
    }
1096719
public void createChunks(String frameKey, byte[] expectedTypes, int chunkId, int totalNumRows, int[] maxVecSizes) throws IOException {
        ab.put1(ExternalFrameHandler.INIT_BYTE);
        ab.put1(ExternalFrameHandler.CREATE_FRAME);
        ab.putStr(frameKey);
        this.expectedTypes = expectedTypes;
        ab.putA1(expectedTypes);
        ab.putA4(maxVecSizes);
        ab.putInt(totalNumRows);
        ab.putInt(chunkId);
        writeToChannel(ab, channel);
    }
809958
public Uploader startUploadFileChunked(int chunkSize, String targetPath, DbxWriteMode writeMode, long numBytes)
    {
        DbxPathV1.checkArg("targetPath", targetPath);
        if (writeMode == null) throw new IllegalArgumentException("'writeMode' can't be null");

        return new ChunkedUploader(targetPath, writeMode, numBytes, new ChunkedUploadOutputStream(chunkSize));
    }
513707
async def _init(self):
        
        chunk = InitChunk()
        chunk.initiate_tag = self._local_verification_tag
        chunk.advertised_rwnd = self._advertised_rwnd
        chunk.outbound_streams = self._outbound_streams_count
        chunk.inbound_streams = self._inbound_streams_max
        chunk.initial_tsn = self._local_tsn
        self._set_extensions(chunk.params)
        await self._send_chunk(chunk)

        # start T1 timer and enter COOKIE-WAIT state
        self._t1_start(chunk)
        self._set_state(self.State.COOKIE_WAIT)
1012229
@NotThreadsafe
    private void initFileStreams(int chunkId) {
        /**
         * {@link Set#add(Object)} returns false if the element already existed in the set.
         * This ensures we initialize the resources for each chunk only once.
         */
        if (chunksHandled.add(chunkId)) {
            try {
                this.indexFileSizeInBytes[chunkId] = 0L;
                this.valueFileSizeInBytes[chunkId] = 0L;
                this.checkSumDigestIndex[chunkId] = CheckSum.getInstance(checkSumType);
                this.checkSumDigestValue[chunkId] = CheckSum.getInstance(checkSumType);
                this.position[chunkId] = 0;
                this.taskIndexFileName[chunkId] = new Path(FileOutputFormat.getOutputPath(conf),
                                                           getStoreName() + "."
                                                                   + Integer.toString(chunkId) + "_"
                                                                   + this.taskId + INDEX_FILE_EXTENSION
                                                                   + fileExtension);
                this.taskValueFileName[chunkId] = new Path(FileOutputFormat.getOutputPath(conf),
                                                           getStoreName() + "."
                                                                   + Integer.toString(chunkId) + "_"
                                                                   + this.taskId + DATA_FILE_EXTENSION
                                                                   + fileExtension);
                if(this.fs == null)
                    this.fs = this.taskIndexFileName[chunkId].getFileSystem(conf);
                if(isValidCompressionEnabled) {
                    this.indexFileStream[chunkId] = new DataOutputStream(new BufferedOutputStream(new GZIPOutputStream(fs.create(this.taskIndexFileName[chunkId]),
                                                                                                                       DEFAULT_BUFFER_SIZE)));
                    this.valueFileStream[chunkId] = new DataOutputStream(new BufferedOutputStream(new GZIPOutputStream(fs.create(this.taskValueFileName[chunkId]),
                                                                                                                       DEFAULT_BUFFER_SIZE)));

                } else {
                    this.indexFileStream[chunkId] = fs.create(this.taskIndexFileName[chunkId]);
                    this.valueFileStream[chunkId] = fs.create(this.taskValueFileName[chunkId]);

                }
                fs.setPermission(this.taskIndexFileName[chunkId],
                                 new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
                logger.info("Setting permission to 755 for " + this.taskIndexFileName[chunkId]);
                fs.setPermission(this.taskValueFileName[chunkId],
                                 new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
                logger.info("Setting permission to 755 for " + this.taskValueFileName[chunkId]);

                logger.info("Opening " + this.taskIndexFileName[chunkId] + " and "
                                    + this.taskValueFileName[chunkId] + " for writing.");
            } catch(IOException e) {
                throw new RuntimeException("Failed to open Input/OutputStream", e);
            }
        }
    }
325928
private void nextChunk() throws IOException {
        if (!bof) {
            readCRLF();
        }
        chunkSize = getChunkSizeFromInputStream(in);
        bof = false;
        pos = 0;
        if (chunkSize == 0) {
            eof = true;
            parseTrailerHeaders();
        }
    }
823781
function (task, chunk, callback) {
        var that = this;
        var formData = new FormData();
        var opt = task.opt;
        var file = opt.file;
        var slice_size = opt.slice_size;
        var session = opt.session;
        var totalSize = file.size;
        var offsetStart = chunk.start;
        var offsetEnd = Math.min(offsetStart + slice_size, totalSize);
        var blob = that.slice.call(file, offsetStart, offsetEnd);
        var chunkSize = blob.size;

        var removeXhr = function (xhr) {
            for (var i = task.uploadingAjax.length - 1; i >= 0; i--) {
                if (xhr === task.uploadingAjax[i]) {
                    task.uploadingAjax.splice(i, 1);
                }
            }
        };
        var preLoaded = 0;
        var updateProgress = function (loaded, immediately) {
            task.loadedSize += loaded - preLoaded;
            preLoaded = loaded;
            task.onTaskProgress && task.onTaskProgress(immediately);
        };
        var uploadChunk = function (cb) {
            formData.append('sliceSize', slice_size);
            formData.append('op', 'upload_slice_data');
            formData.append('session', session);
            formData.append('offset', offsetStart);
            opt.sha && formData.append('sha', opt.sha);
            formData.append('fileContent', blob);

            that.getAppSign(function (sign) {
                opt.sign = sign;
                var url = that.getCgiUrl(opt.path, opt.sign);

                var ajax = $.ajax({
                    type: 'POST',
                    dataType: "JSON",
                    url: url,
                    data: formData,
                    xhr: function () {
                        var xhr = $.ajaxSettings.xhr();
                        xhr.upload.onprogress = function (evt) {
                            updateProgress(evt.loaded);
                            task.onTaskProgress && task.onTaskProgress();
                        };
                        return xhr;
                    },
                    success: function (res) {
                        updateProgress(chunkSize, true);
                        res = res || {};
                        if (res.code === 0) {
                            cb(null, res);
                        } else {
                            cb('error', res);
                        }
                    },
                    error: function () {
                        updateProgress(0, true);
                        cb('error');
                    },
                    complete: function () {
                        removeXhr(ajax);
                    },
                    processData: false,
                    contentType: false
                });
                task.uploadingAjax.push(ajax);
            });
        };

        // 失败重试 3 次
        var tryUpload = function (times) {
            if (opt.globalTask.state === 'cancel') return;
            uploadChunk(function (err, data) {
                if (err) { // fail, retry
                    if (times >= that.uploadMaxRetryTimes || task.uploadError || opt.globalTask.state === 'cancel') {
                        callback(err, data);
                    } else {
                        tryUpload(times + 1);
                    }
                } else { // success
                    callback(err, data);
                }
            });
        };
        fileSliceCopy(file, offsetStart, offsetEnd, function (newBlob) {
            blob = newBlob;
            chunkSize = blob.size;
            tryUpload(1);
        });

    }
967116
function hotDownloadUpdateChunk(chunkId) {
		var script = document.createElement("script");
		script.charset = "utf-8";
		script.src = $require$.p + $hotChunkFilename$;
		if ($crossOriginLoading$) script.crossOrigin = $crossOriginLoading$;
		document.head.appendChild(script);
	}
1309082
def __upload_chunk(self, resource, chunk_size, bytes, bytes_start, bytes_read):
        """"""

        # note: string conversion required here due to open encoding bug in requests-oauthlib.
        headers = {
            'content-type': self.content_type,
            'content-length': str(min([chunk_size, self._file_size - bytes_read])),
            'content-range': "bytes {0}-{1}/{2}".format(
                bytes_start, bytes_read - 1, self._file_size)
        }

        return Request(self._client, 'put', resource,
                       domain=self._DEFAULT_DOMAIN, headers=headers, body=bytes).perform()
651966
private ServletRequest init() throws MultipartException, IOException {
		// retrieve multipart/form-data parameters
		if (Multipart.isMultipartContent(request)) {
			Multipart multipart = new Multipart();
			multipart.parse(request, new PartHandler() {

				@Override
				public void handleFormItem(String name, String value) {
					multipartParams.put( name, value );
				}

				@Override
				public void handleFileItem(String name, FileItem fileItem) {
					files.add(fileItem);
				}

			});
		}

		return this;
	}