Number of bytes we have downloaded and verified.
540986
def completedLength
      num = @completeBlocks.countSet
      # Last block may be smaller
      extra = 0
      if @completeBlocks.set?(@completeBlocks.length-1)
        num -= 1
        extra = @lastBlockLength
      end
      num*@blockSize + extra
    end
1039762
private void checkUserFileBufferBytes() {
    if (!isSet(PropertyKey.USER_FILE_BUFFER_BYTES)) { // load from hadoop conf
      return;
    }
    long usrFileBufferBytes = getBytes(PropertyKey.USER_FILE_BUFFER_BYTES);
    Preconditions.checkState((usrFileBufferBytes & Integer.MAX_VALUE) == usrFileBufferBytes,
        PreconditionMessage.INVALID_USER_FILE_BUFFER_BYTES.toString(),
        PropertyKey.Name.USER_FILE_BUFFER_BYTES, usrFileBufferBytes);
  }
473247
def filesize(self):
        """"""
        if self.kind == 'data':
            return len(self._data.data)
        return self._data.filesize
543178
def human_bytes(n):
    
    if n < 1024:
        return '%d B' % n
    k = n/1024
    if k < 1024:
        return '%d KB' % round(k)
    m = k/1024
    if m < 1024:
        return '%.1f MB' % m
    g = m/1024
    return '%.2f GB' % g
501052
def verify_uploaded_file(
            self,
            destination_folder_id,
            source_path,
            verbose = True,
    ):
        '''
        
        '''
        source_file_size = os.stat(source_path).st_size

        total_part_size = 0
        file_position = 0
        uploaded_box_file_ids = self.find_file( destination_folder_id, os.path.basename( source_path ) )
        total_sha1 = hashlib.sha1()
        for i, file_id in enumerate(uploaded_box_file_ids):
            file_info = self.client.file( file_id = file_id ).get()
            uploaded_sha1 = file_info.response_object['sha1']
            uploaded_size = file_info.response_object['size']

            part_sha1 = read_sha1( source_path, start_byte = file_position, read_size = uploaded_size, extra_hashers = [total_sha1] )
            if part_sha1.hexdigest() != uploaded_sha1:
                print( '\n' )
                print( 'Part sha1: ' + part_sha1.hexdigest() )
                print( 'Uploaded sha1: ' + uploaded_sha1 )
                print('Sha1 hash of uploaded file {0} ({1}) does not match'.format(file_info.response_object['name'], file_id) )
                return False

            file_position += uploaded_size
            total_part_size += uploaded_size
            if len(uploaded_box_file_ids) > 1:
                print( 'Finished verifying part {0} of {1} of {2}'.format( i+1, len(uploaded_box_file_ids), file_id ) )

        assert( source_file_size == total_part_size )

        if verbose:
            print( 'Verified uploaded file {0} ({1}) with sha1: {2}'.format(source_path, file_id, total_sha1.hexdigest()) )

        return True
919380
def GetByteSize(self):
    
    if self.size == definitions.SIZE_NATIVE or self.units != 'bytes':
      return None

    return self.size
48499
def _log_progress(self, bytes_downloaded):
    
    self._total_bytes_downloaded += bytes_downloaded
    now = time.time()
    if (self._interactive_mode() or
        now - self._last_progress_msg_print_time > 15):
      # Print progress message every 15 secs or if interactive progress
      # tracking is enabled.
      self._print_download_progress_msg(
          "Downloading %s: %s" % (self._url,
                                  tf_utils.bytes_to_readable_str(
                                      self._total_bytes_downloaded, True)))
      self._last_progress_msg_print_time = now
780522
def get_copies_count(self):
        
        setup = api.get_setup()
        default_num = setup.getDefaultNumberOfCopies()
        request_num = self.request.form.get("copies_count")
        return to_int(request_num, default_num)
344689
def Size(self):
        
        return s.uint32 + 12 + s.uint32 + s.uint32 + len(self.Payload)
1370491
def main(config, log):
    
    validate(config)
    paths_and_urls = get_urls(config)
    if not paths_and_urls:
        log.warning('No artifacts; nothing to download.')
        return

    # Download files.
    total_size = 0
    chunk_size = max(min(max(v[1] for v in paths_and_urls.values()) // 50, 1048576), 1024)
    log.info('Downloading file%s (1 dot ~ %d KiB):', '' if len(paths_and_urls) == 1 else 's', chunk_size // 1024)
    for size, local_path, url in sorted((v[1], k, v[0]) for k, v in paths_and_urls.items()):
        download_file(config, local_path, url, size, chunk_size)
        total_size += size
        if config['mangle_coverage']:
            mangle_coverage(local_path)

    log.info('Downloaded %d file(s), %d bytes total.', len(paths_and_urls), total_size)
1260950
def _on_bytes_read(self, num_bytes_read):
        
        self.actual_bytes_read += num_bytes_read
        if self.actual_bytes_read > self.bytes_to_read:
            raise TooLargeChunkDownloadError(self.actual_bytes_read, self.bytes_to_read, self.local_path)
        self.download_context.send_processed_message(num_bytes_read)
298522
def bytes(self):
        """"""
        flagByte = 0x00
        messageType = 0
        if self._messageType is not None:
            messageType = self._messageType << 5
        extendedBit = 0 if self._extended is None else self._extended << 4
        hopsMax = 0 if self._hopsMax is None else self._hopsMax
        hopsLeft = 0 if self._hopsLeft is None else (self._hopsLeft << 2)
        flagByte = flagByte | messageType | extendedBit | hopsLeft | hopsMax
        return bytes([flagByte])
1316784
def determine_bytes_per_chunk(self, size):
        
        workers = self.settings.config.download_workers
        if not workers or workers == 'None':
            workers = 1
        bytes_per_chunk = int(math.ceil(size / float(workers)))
        if bytes_per_chunk < self.bytes_per_chunk:
            bytes_per_chunk = self.bytes_per_chunk
        return bytes_per_chunk
1067905
def number_of_modified_bytes(buf, fuzzed_buf):
    
    count = 0
    for idx, b in enumerate(buf):
        if b != fuzzed_buf[idx]:
            count += 1
    return count
1089563
def handle_log_data(self, m):
        ''''''
        if self.download_file is None:
            return
        # lose some data
        # import random
        # if random.uniform(0,1) < 0.05:
        #    print('dropping ', str(m))
        #    return
        if m.ofs != self.download_ofs:
            self.download_file.seek(m.ofs)
            self.download_ofs = m.ofs
        if m.count != 0:
            data = m.data[:m.count]
            s = ''.join(str(chr(x)) for x in data)
            self.download_file.write(s)
            self.download_set.add(m.ofs // 90)
            self.download_ofs += m.count
        self.download_last_timestamp = time.time()
        if m.count == 0 or (m.count < 90 and len(self.download_set) == 1 + (m.ofs // 90)):
            dt = time.time() - self.download_start
            self.download_file.close()
            size = os.path.getsize(self.download_filename)
            speed = size / (1000.0 * dt)
            print("Finished downloading %s (%u bytes %u seconds, %.1f kbyte/sec %u retries)" % (
                self.download_filename,
                size,
                dt, speed,
                self.retries))
            self.download_file = None
            self.download_filename = None
            self.download_set = set()
1047985
def check_download(obj, *args, **kwargs):
    """"""
    version = args[0]
    workdir = args[1]
    signame = args[2]
    if version:
        local_version = get_local_version(workdir, signame)
        if not verify_sigfile(workdir, signame) or version != local_version:
            error("[-] \033[91mFailed to verify signature: %s from: %s\033[0m"
                  % (signame, obj.url))
            raise ValueError('Failed to verify signature: %s' % signame)
597167
def _verify_file_size(self):
        
        if self._file_size > PartSize.MAXIMUM_OBJECT_SIZE:
            self._status = TransferState.FAILED
            raise SbgError('File size = {}b. Maximum file size is {}b'.format(
                self._file_size, PartSize.MAXIMUM_OBJECT_SIZE)
            )