计算输入流的etag:

    etag规格参考 https://developer.qiniu.com/kodo/manual/1231/appendix#3

    Args:
        input_stream: 待计算etag的二进制流

    Returns:
        输入流的etag值
973719
def etag_stream(input_stream):
    
    array = [_sha1(block) for block in _file_iter(input_stream, _BLOCK_SIZE)]
    if len(array) == 0:
        array = [_sha1(b'')]
    if len(array) == 1:
        data = array[0]
        prefix = b'\x16'
    else:
        sha1_str = b('').join(array)
        data = _sha1(sha1_str)
        prefix = b'\x96'
    return urlsafe_base64_encode(prefix + data)
872596
function (input, encoding) {
        var module = _gpfEncodings[encoding];
        if (undefined === module) {
            gpf.Error.encodingNotSupported();
        }
        return new EncoderStream(module[0], input);
    }
1218525
def getinputencoding(stream=None):
    
    if stream is None:
        stream = sys.stdin
    encoding = stream.encoding
    if not encoding:
        encoding = getpreferredencoding()
    return encoding
1285377
private static String inputStreamToString(InputStream stream)
			throws IOException {
		ByteArrayOutputStream result = new ByteArrayOutputStream();
		byte[] buffer = new byte[1024];
		int length;
		String resultString = null;
		while ((length = stream.read(buffer)) != -1) {
			result.write(buffer, 0, length);
		}
		resultString = result.toString("UTF-8");
		return resultString;
	}
1137924
def stream_parameters(streams, no_duplicates=True, exclude=['name']):
    
    param_groups = []
    for s in streams:
        if not s.contents and isinstance(s.hashkey, dict):
            param_groups.append(list(s.hashkey))
        else:
            param_groups.append(list(s.contents))
    names = [name for group in param_groups for name in group]

    if no_duplicates:
        clashes = sorted(set([n for n in names if names.count(n) > 1]))
        clash_streams = []
        for s in streams:
            for c in clashes:
                if c in s.contents or (not s.contents and isinstance(s.hashkey, dict) and c in s.hashkey):
                    clash_streams.append(s)
        if clashes:
            clashing = ', '.join([repr(c) for c in clash_streams[:-1]])
            raise Exception('The supplied stream objects %s and %s '
                            'clash on the following parameters: %r'
                            % (clashing, clash_streams[-1], clashes))
    return [name for name in names if name not in exclude]
451273
def _extract_etag(response):
    '''  '''
    if response and response.headers:
        for name, value in response.headers:
            if name.lower() == 'etag':
                return value

    return None
345511
function etag (entity, options) {
  if (entity == null) {
    throw new TypeError('argument entity is required')
  }

  // support fs.Stats object
  var isStats = isstats(entity)
  var weak = options && typeof options.weak === 'boolean'
    ? options.weak
    : isStats

  // validate argument
  if (!isStats && typeof entity !== 'string' && !Buffer.isBuffer(entity)) {
    throw new TypeError('argument entity must be string, Buffer, or fs.Stats')
  }

  // generate entity tag
  var tag = isStats
    ? stattag(entity)
    : entitytag(entity)

  return weak
    ? 'W/' + tag
    : tag
}
1057039
def get_input_ext(self, input_file):
        
        if input_file.endswith(".bam"):
            input_ext = ".bam"
        elif input_file.endswith(".fastq.gz") or input_file.endswith(".fq.gz"):
            input_ext = ".fastq.gz"
        elif input_file.endswith(".fastq") or input_file.endswith(".fq"):
            input_ext = ".fastq"
        else:
            errmsg = "'{}'; this pipeline can only deal with .bam, .fastq, " \
                     "or .fastq.gz files".format(input_file)
            raise UnsupportedFiletypeException(errmsg)
        return input_ext
48787
def fetch(self, output_path, last_etag=None, lastmodified=None, progress_bar=False):
        
        req_headers = {}
        usr_agent = get_user_agent()
        if usr_agent is not None:
            req_headers['User-Agent'] = usr_agent

        if last_etag is not None:
            req_headers['If-None-Match'] = '"' + last_etag + '"'
        if lastmodified is not None:
            req_headers['If-Modified-Since'] = lastmodified

        response = requests.get(self.url, headers=req_headers, stream=progress_bar)

        if response.status_code == 304:
            # Conditions didn't match, no content returned
            return None
        if response.status_code != 200:
            raise requests.HTTPError

        with open(output_path, 'wb') as f:
            if progress_bar:
                total_size = long(response.headers.get('content-length'))
                for chunk in progress.bar(response.iter_content(chunk_size=self.chunk_size),
                                          expected_size=(total_size / self.chunk_size) + 1):
                    if chunk:
                        f.write(chunk)
                        f.flush()
            else:
                f.write(response.content)

        headers = normalize_headers(response.headers)

        if 'etag' not in headers:
            headers['etag'] = fs_util.calc_file_md5(output_path)

        return headers
1147332
def init_with_stream(stream)
      if stream.eof?
        raise ArgumentError, "Can't parse transaction input from stream because it is already closed."
      end

      if !(@previous_hash = stream.read(32)) || @previous_hash.bytesize != 32
        raise ArgumentError, "Failed to read 32-byte previous_hash from stream."
      end

      if !(@previous_index = BTC::WireFormat.read_uint32le(stream: stream).first)
        raise ArgumentError, "Failed to read previous_index from stream."
      end

      is_coinbase = (@previous_hash == ZERO_HASH256 && @previous_index == INVALID_INDEX)

      if !(scriptdata = BTC::WireFormat.read_string(stream: stream).first)
        raise ArgumentError, "Failed to read signature_script data from stream."
      end

      @coinbase_data = nil
      @signature_script = nil

      if is_coinbase
        @coinbase_data = scriptdata
      else
        @signature_script = BTC::Script.new(data: scriptdata)
      end

      if !(@sequence = BTC::WireFormat.read_uint32le(stream: stream).first)
        raise ArgumentError, "Failed to read sequence from stream."
      end
    end
1010627
def gen_etag():
    """"""

    my_salt = gen_uuid()
    if six.PY2:
        my_salt = my_salt.decode('utf-8')
    elif six.PY3:
        my_salt = my_salt.encode('utf-8')
    md5 = hashlib.md5()
    md5.update(my_salt)
    return md5.hexdigest()
667613
def parse(input_: Union[str, FileStream], source: str) -> Optional[str]:
    

    # Step 1: Tokenize the input stream
    error_listener = ParseErrorListener()
    if not isinstance(input_, FileStream):
        input_ = InputStream(input_)
    lexer = jsgLexer(input_)
    lexer.addErrorListener(error_listener)
    tokens = CommonTokenStream(lexer)
    tokens.fill()
    if error_listener.n_errors:
        return None

    # Step 2: Generate the parse tree
    parser = jsgParser(tokens)
    parser.addErrorListener(error_listener)
    parse_tree = parser.doc()
    if error_listener.n_errors:
        return None

    # Step 3: Transform the results the results
    parser = JSGDocParser()
    parser.visit(parse_tree)

    if parser.undefined_tokens():
        for tkn in parser.undefined_tokens():
            print("Undefined token: " + tkn)
        return None

    return parser.as_python(source)
567988
def iter_csv_stream(input_stream, fieldnames=None, sniff=False, *args, **kwargs):
    '''  '''
    if 'dialect' not in kwargs and sniff:
        kwargs['dialect'] = csv.Sniffer().sniff(input_stream.read(1024))
        input_stream.seek(0)
    if 'quoting' in kwargs and kwargs['quoting'] is None:
        kwargs['quoting'] = csv.QUOTE_MINIMAL
    if fieldnames:
        # read csv using dictreader
        if isinstance(fieldnames, bool):
            reader = csv.DictReader(input_stream, *args, **kwargs)
        else:
            reader = csv.DictReader(input_stream, *args, fieldnames=fieldnames, **kwargs)
        for row in reader:
            yield row
    else:
        csvreader = csv.reader(input_stream, *args, **kwargs)
        for row in csvreader:
            yield row
1042902
def stream_keys(self, bucket, timeout=None):
        
        msg_code = riak.pb.messages.MSG_CODE_LIST_KEYS_REQ
        codec = self._get_codec(msg_code)
        msg = codec.encode_stream_keys(bucket, timeout)
        self._send_msg(msg.msg_code, msg.data)
        return PbufKeyStream(self, codec)
138535
def get_validate_upload_form_kwargs(self):
        

        kwargs = {
            'storage': self.get_storage(),
            'upload_to': self.get_upload_to(),
            'content_type_prefix': self.get_content_type_prefix(),
            'process_to': self.get_process_to(),
            'processed_key_generator': self.get_processed_key_generator(),
        }

        # ``data`` may be provided by a POST from the JavaScript if using a
        # DropZone form, or as querystrings on a redirect GET request from
        # Amazon if not.
        data = {
            'bucket_name': self._get_bucket_name(),
            'key_name': self._get_key_name(),
            'etag': self._get_etag(),
        }
        kwargs.update({'data': data})
        return kwargs
815006
def getoutputencoding(stream=None):
    
    if stream is None:
        stream = sys.stdout
    encoding = stream.encoding
    if not encoding:
        encoding = getpreferredencoding()
    return encoding
315934
def get_streaming(self, path, stype="M3U8_AUTO_480", **kwargs):
        

        params = {
            'path': path,
            'type': stype
        }
        url = 'https://{0}/rest/2.0/pcs/file'.format(BAIDUPCS_SERVER)
        while True:
            ret = self._request('file', 'streaming', url=url, extra_params=params, **kwargs)
            if not ret.ok:
                logging.debug('get_streaming ret_status_code %s' % ret.status_code)
                jdata = json.loads(ret.content)
                if jdata['error_code'] == 31345:
                    # 再试一次
                    continue
                elif jdata['error_code'] == 31066:
                    # 文件不存在
                    return 31066
                elif jdata['error_code'] == 31304:
                    # 文件类型不支持
                    return 31304
                elif jdata['error_code'] == 31023:
                    # params error
                    return 31023
            return ret.content