chunk: download, verify, save, change index, repeat
553004
function syncThroughChunks (prevChunk) {
    // all already synced?
    if (index > networkIndex) {
      return deferred.resolve()
    }

    function cachedGetHeader (id) {
      if (prevChunk === undefined ||
          !_.isNumber(id) ||
          Math.floor(id / 2016) !== index - 1) {
        return self.getHeader(id)
      }

      var idx = id % 2016
      var rawHeader = prevChunk.slice(idx * 80, (idx + 1) * 80)
      return Promise.resolve(_.extend(util.buffer2header(rawHeader), {
        height: id,
        hash: util.hashEncode(util.sha256x2(rawHeader))
      }))
    }

    // get not verified chunk for index
    self.connector.headersQuery(index * 2016, {count: 2016})
      .then(function (res) {
        var chunkHex = res.headers
        var prevHeaderPromise = index === 0
          ? {hash: util.zfill('', 64)} // special case for zero header
          : cachedGetHeader(index * 2016 - 1)

        return Promise.all([prevHeaderPromise, new Buffer(chunkHex, 'hex')])
      })
      .spread(function (prevHeader, rawChunk) {
        // compare hashPrevBlock of first header from chunk
        //  and derease index for blockchain reorg if not equal
        var firstHeader = util.buffer2header(rawChunk.slice(0, 80))
        if (firstHeader.hashPrevBlock !== prevHeader.hash) {
          index -= 1
          return timers.setImmediate(syncThroughChunks)
        }

        // remember for verifyHeader
        var prevHeaderHash = prevHeader.hash

        // calculate hash of last header from chunk for blockchain and storage
        var lastHash = util.hashEncode(util.sha256x2(rawChunk.slice(-80)))

        // calculate target for index
        return getTarget(index, headersChain, cachedGetHeader)
          .then(function (target) {
            // verify headers in chunk
            _.range(0, rawChunk.length, 80).forEach(function (offset) {
              var rawHeader = rawChunk.slice(offset, offset + 80)
              var currentHash = util.hashEncode(util.sha256x2(rawHeader))
              var currentHeader = util.buffer2header(rawHeader)

              verifyHeader(currentHash, currentHeader,
                           prevHeaderHash, prevHeader, target, self._isTestnet)

              prevHeaderHash = currentHash
              prevHeader = currentHeader
            })

            // set last hash to storage
            var promises = [self.storage.setLastHash(lastHash)]

            // truncate chunk hashes and headers if compact mode supported
            if (self.storage.compactMode) {
              promises.push(self.storage.truncateChunkHashes(index))
              promises.push(self.storage.truncateHeaders(0))
            }
            // truncate headers if compact mode not supported
            //  affected only when reorg needed
            if (!self.storage.compactMode) {
              promises.push(self.storage.truncateHeaders(index * 2016))
            }

            // wait all storage queries
            return Promise.all(promises)
          })
          .then(function () {
            // save chunk hash or headers
            if (self.storage.compactMode && rawChunk.length === 2016 * 80) {
              var chunkHash = util.hashEncode(util.sha256x2(rawChunk))
              return self.storage.putChunkHashes([chunkHash])

            } else {
              var headers = _.range(0, rawChunk.length, 80).map(function (offset) {
                return rawChunk.slice(offset, offset + 80).toString('hex')
              })
              return self.storage.putHeaders(headers)

            }
          })
          .then(function () {
            // update block hash and height for blockchain
            self.latest = {
              hash: lastHash,
              height: index * 2016 + rawChunk.length / 80 - 1
            }
            self.emit('newBlock', self.latest.hash, self.latest.height)

            // increase chunk index
            index += 1

            // sync next chunk
            timers.setImmediate(syncThroughChunks, rawChunk)
          })
      })
      .catch(function (err) { deferred.reject(err) })
  }
707471
def connectChunk(key, chunk):
    
    upLinks = []
    schunk = chunk[0].strip().split()

    for idx in range(4, len(schunk)):
        upLinks.append(schunk[idx])

    result = {'link': schunk[1],
              'downLink': schunk[2],
              'numUpLinks': schunk[3],
              'upLinks': upLinks}

    return result
331193
def chunks(self):
        

        while True:
            t, v = self.chunk()
            yield t, v
            if t == b'IEND':
                break
180412
def chunk(self, count):
        
        page = 1
        results = self.for_page(page, count).get()

        while results:
            yield results

            page += 1

            results = self.for_page(page, count).get()
407598
def _chunk_offsets(self):
        """"""
        index = 0
        blob_size = self.blob_properties.get('content-length')
        while index < blob_size:
            yield index
            index = index + self._chunk_size
509825
def download_file_part_run(download_context):
    
    destination_dir, file_url_data_dict, seek_amt, bytes_to_read = download_context.params
    project_file = ProjectFile(file_url_data_dict)
    local_path = project_file.get_local_path(destination_dir)
    retry_chunk_downloader = RetryChunkDownloader(project_file, local_path,
                                                  seek_amt, bytes_to_read,
                                                  download_context)
    retry_chunk_downloader.run()
    return 'ok'
434574
def _finalize_chunk(self, dd, offsets):
        # type: (Downloader, blobxfer.models.download.Descriptor,
        #        blobxfer.models.download.Offsets) -> None
        
        if dd.entity.is_encrypted:
            dd.mark_unchecked_chunk_decrypted(offsets.chunk_num)
        # integrity check data and write to disk (this is called
        # regardless of md5/hmac enablement for resume purposes)
        dd.perform_chunked_integrity_check()
        # remove from disk set and add bytes to counter
        with self._disk_operation_lock:
            self._disk_set.remove(
                blobxfer.operations.download.Downloader.
                create_unique_disk_operation_id(dd, offsets))
            self._download_bytes_sofar += offsets.num_bytes
152633
function readNextChunk() {

            // If no further chunks remain, inform of completion and stop
            if (offset >= blob.size) {

                // Fire completion event for completed blob
                if (guacWriter.oncomplete)
                    guacWriter.oncomplete(blob);

                // No further chunks to read
                return;

            }

            // Obtain reference to next chunk as a new blob
            var chunk = slice(blob, offset, offset + arrayBufferWriter.blobLength);
            offset += arrayBufferWriter.blobLength;

            // Attempt to read the blob contents represented by the blob into
            // a new array buffer
            reader.readAsArrayBuffer(chunk);

        }
505036
function Chunk(options)
{
  // make sure the options object contains all the necessary information
  Errors.assertInternal(Util.isObject(options));
  Errors.assertInternal(Util.isObject(options.statement));
  Errors.assertInternal(Util.isObject(options.services));
  Errors.assertInternal(Util.isNumber(options.startIndex));
  Errors.assertInternal(Util.isArray(options.columns));
  Errors.assertInternal(Util.isObject(options.mapColumnNameToIndices));
  Errors.assertInternal(Util.isObject(options.statementParameters));
  Errors.assertInternal(Util.isString(options.resultVersion));
  Errors.assertInternal(Util.isNumber(options.rowCount));

  // if the result is small (i.e. not persisted on S3/Blob), there's no
  // compressed and uncompressed size, so default to -1
  this._compressedSize   = options.compressedSize   || -1;
  this._uncompressedSize = options.uncompressedSize || -1;

  // copy out other information from the options object and save it
  this._statement              = options.statement;
  this._services               = options.services;
  this._startIndex             = options.startIndex;
  this._url                    = options.url;
  this._columns                = options.columns;
  this._mapColumnNameToIndices = options.mapColumnNameToIndices;
  this._chunkHeaders           = options.chunkHeaders;
  this._rowset                 = options.rowset;

  // use the start index and row count to compute the end index
  this._endIndex = this._startIndex + options.rowCount - 1;

  // use the start and end index to build an id for this chunk
  this._id = buildId(this._startIndex, this._endIndex);
}
644069
def process_chunks(self, chunks):
        
        chunk, = chunks
        if chunk.keys:
            self.result[chunk.keys] = chunk.data
        else:
            self.result[...] = chunk.data
1370099
def verify_chunks(self, chunks):
        '''
        
        '''
        err = []
        for chunk in chunks:
            err.extend(self.verify_data(chunk))
        return err
1282014
def save_as(self, filename):
        
        with open(filename, 'w+b') as f:
            for chunk in self._download():
                f.write(chunk)
267755
def http_stream(url, chunk=4096):
    
    ret = requests.get(url, stream=True)
    ret.raise_for_status()
    for data in ret.iter_content(chunk):
        yield data
585758
function download(input, output) {
	if (!output) output = input.split("/").pop();
	var file = fs.createWriteStream(output);
	var request = http.get(input, response => { response.pipe(file); });
}
1206019
def __flush_data(self, data):
        
        # Ensure the index, even if there's nothing to write, so
        # the filemd5 command always succeeds.
        self.__ensure_indexes()
        self._file['md5'].update(data)

        if not data:
            return
        assert(len(data) <= self.chunk_size)

        chunk = {"files_id": self._file["_id"],
                 "n": self._chunk_number,
                 "data": Binary(data)}

        try:
            self._chunks.insert_one(chunk)
        except DuplicateKeyError:
            self._raise_file_exists(self._file['_id'])
        self._chunk_number += 1
        self._position += len(data)
233008
def receive_data_chunk(self, raw_data, start):
        
        self.file.write(raw_data)
        # CHANGED: This un-hangs us long enough to keep things rolling.
        eventlet.sleep(0)
1265156
private void writeChunk(String spaceId, ChunkInputStream chunk)
        throws NotFoundException {
        // Write chunk as a temp file
        String chunkId = chunk.getChunkId();
        File chunkFile = IOUtil.writeStreamToFile(chunk);

        try {
            String chunkChecksum = getChunkChecksum(chunkFile);

            // Write chunk if it is not already in storage (or jumpstart is enabled)
            if (jumpStart || !chunkInStorage(spaceId, chunkId, chunkChecksum)) {
                try {
                    createRetrier().execute(new Retriable() {
                        private int attempt = 0;

                        @Override
                        public Object retry() throws Exception {
                            attempt++;
                            try (InputStream chunkStream = new FileInputStream(chunkFile)) {
                                ChunkInputStream chunkFileStream =
                                    new ChunkInputStream(chunkId,
                                                         chunkStream,
                                                         chunkFile.length(),
                                                         chunk.md5Preserved());
                                writeSingle(spaceId, chunkChecksum, chunkFileStream, attempt == getMaxRetries() + 1);
                            }
                            return "";
                        }
                    });

                } catch (Exception e) {
                    String err = "Failed to store chunk with ID " + chunkId +
                                 " in space " + spaceId + " after " + getMaxRetries() +
                                 " attempts. Last error: " + e.getMessage();
                    throw new DuraCloudRuntimeException(err, e);
                }
            }
        } finally {
            if (null != chunkFile && chunkFile.exists()) {
                FileUtils.deleteQuietly(chunkFile);
            }
        }
    }