Do our best to decode a Python source file correctly.
960613
def _read_python_source(self, filename):
        
        try:
            f = open(filename, "rb")
        except IOError as err:
            self.log_error("Can't open %s: %s", filename, err)
            return None, None
        try:
            encoding = tokenize.detect_encoding(f.readline)[0]
        finally:
            f.close()
        with _open_with_encoding(filename, "r", encoding=encoding) as f:
            return _from_system_newlines(f.read()), encoding
1194425
def get_source(self, name):
        """"""
        path = self.get_filename(name)
        try:
            source_bytes = self.get_data(path)
        except OSError as exc:
            e = _ImportError('source not available through get_data()',
                             name=name)
            e.__cause__ = exc
            raise e
        return decode_source(source_bytes)
292466
def get_best_encoding(stream):
    """"""
    rv = getattr(stream, 'encoding', None) or sys.getdefaultencoding()
    if is_ascii_encoding(rv):
        return 'utf-8'
    return rv
984551
def decode(text):
    
    try:
        if text.startswith(BOM_UTF8):
            # UTF-8 with BOM
            return to_text_string(text[len(BOM_UTF8):], 'utf-8'), 'utf-8-bom'
        elif text.startswith(BOM_UTF16):
            # UTF-16 with BOM
            return to_text_string(text[len(BOM_UTF16):], 'utf-16'), 'utf-16'
        elif text.startswith(BOM_UTF32):
            # UTF-32 with BOM
            return to_text_string(text[len(BOM_UTF32):], 'utf-32'), 'utf-32'
        coding = get_coding(text)
        if coding:
            return to_text_string(text, coding), coding
    except (UnicodeError, LookupError):
        pass
    # Assume UTF-8
    try:
        return to_text_string(text, 'utf-8'), 'utf-8-guessed'
    except (UnicodeError, LookupError):
        pass
    # Assume Latin-1 (behaviour before 3.7.1)
    return to_text_string(text, "latin-1"), 'latin-1-guessed'
182583
def auto_decode(data):
    # type: (bytes) -> Text
    """"""
    for bom, encoding in BOMS:
        if data.startswith(bom):
            return data[len(bom):].decode(encoding)
    # Lets check the first two lines as in PEP263
    for line in data.split(b'\n')[:2]:
        if line[0:1] == b'#' and ENCODING_RE.search(line):
            encoding = ENCODING_RE.search(line).groups()[0].decode('ascii')
            return data.decode(encoding)
    return data.decode(
        locale.getpreferredencoding(False) or sys.getdefaultencoding(),
    )
1064478
def read_python_code(filename):
    ""
    file = open(filename, "rt")
    try:
        source = file.read()
    finally:
        file.close()
    # Python 2.6 and below did not support passing strings to exec() &
    # compile() functions containing line separators other than '\n'. To
    # support them we need to manually make sure such line endings get
    # converted even on platforms where this is not handled by native text file
    # read operations.
    source = source.replace("\r\n", "\n").replace("\r", "\n")
    return compile(source, filename, "exec")
634277
def decode(cls, line):
        
        if line.encoded:
            encoding = getattr(line, 'encoding_param', None)
            if encoding and encoding.upper() == cls.base64string:
                line.value = codecs.decode(self.value.encode("utf-8"), "base64").decode(encoding)
            else:
                line.value = stringToTextValues(line.value)[0]
            line.encoded=False
331543
def decode_from_file(estimator,
                     filename,
                     hparams,
                     decode_hp,
                     decode_to_file=None,
                     checkpoint_path=None):
  """"""
  if not decode_hp.batch_size:
    decode_hp.batch_size = 32
    tf.logging.info(
        "decode_hp.batch_size not specified; default=%d" % decode_hp.batch_size)

  # Inputs vocabulary is set to targets if there are no inputs in the problem,
  # e.g., for language models where the inputs are just a prefix of targets.
  p_hp = hparams.problem_hparams
  has_input = "inputs" in p_hp.vocabulary
  inputs_vocab_key = "inputs" if has_input else "targets"
  inputs_vocab = p_hp.vocabulary[inputs_vocab_key]
  targets_vocab = p_hp.vocabulary["targets"]
  problem_name = FLAGS.problem
  filename = _add_shard_to_filename(filename, decode_hp)
  tf.logging.info("Performing decoding from file (%s)." % filename)
  if has_input:
    sorted_inputs, sorted_keys = _get_sorted_inputs(
        filename, decode_hp.delimiter)
  else:
    sorted_inputs = _get_language_modeling_inputs(
        filename, decode_hp.delimiter, repeat=decode_hp.num_decodes)
    sorted_keys = range(len(sorted_inputs))
  num_sentences = len(sorted_inputs)
  num_decode_batches = (num_sentences - 1) // decode_hp.batch_size + 1

  if estimator.config.use_tpu:
    length = getattr(hparams, "length", 0) or hparams.max_length
    batch_ids = []
    for line in sorted_inputs:
      if has_input:
        ids = inputs_vocab.encode(line.strip()) + [1]
      else:
        ids = targets_vocab.encode(line)
      if len(ids) < length:
        ids.extend([0] * (length - len(ids)))
      else:
        ids = ids[:length]
      batch_ids.append(ids)
    np_ids = np.array(batch_ids, dtype=np.int32)
    def input_fn(params):
      batch_size = params["batch_size"]
      dataset = tf.data.Dataset.from_tensor_slices({"inputs": np_ids})
      dataset = dataset.map(
          lambda ex: {"inputs": tf.reshape(ex["inputs"], (length, 1, 1))})
      dataset = dataset.batch(batch_size)
      return dataset
  else:
    def input_fn():
      input_gen = _decode_batch_input_fn(
          num_decode_batches, sorted_inputs,
          inputs_vocab, decode_hp.batch_size,
          decode_hp.max_input_size,
          task_id=decode_hp.multiproblem_task_id, has_input=has_input)
      gen_fn = make_input_fn_from_generator(input_gen)
      example = gen_fn()
      return _decode_input_tensor_to_features_dict(example, hparams)
  decodes = []
  result_iter = estimator.predict(input_fn, checkpoint_path=checkpoint_path)

  start_time = time.time()
  total_time_per_step = 0
  total_cnt = 0

  def timer(gen):
    while True:
      try:
        start_time = time.time()
        item = next(gen)
        elapsed_time = time.time() - start_time
        yield elapsed_time, item
      except StopIteration:
        break

  for elapsed_time, result in timer(result_iter):
    if decode_hp.return_beams:
      beam_decodes = []
      beam_scores = []
      output_beams = np.split(result["outputs"], decode_hp.beam_size, axis=0)
      scores = None
      if "scores" in result:
        if np.isscalar(result["scores"]):
          result["scores"] = result["scores"].reshape(1)
        scores = np.split(result["scores"], decode_hp.beam_size, axis=0)
      for k, beam in enumerate(output_beams):
        tf.logging.info("BEAM %d:" % k)
        score = scores and scores[k]
        _, decoded_outputs, _ = log_decode_results(
            result["inputs"],
            beam,
            problem_name,
            None,
            inputs_vocab,
            targets_vocab,
            log_results=decode_hp.log_results,
            skip_eos_postprocess=decode_hp.skip_eos_postprocess)
        beam_decodes.append(decoded_outputs)
        if decode_hp.write_beam_scores:
          beam_scores.append(score)
      if decode_hp.write_beam_scores:
        decodes.append("\t".join([
            "\t".join([d, "%.2f" % s])
            for d, s in zip(beam_decodes, beam_scores)
        ]))
      else:
        decodes.append("\t".join(beam_decodes))
    else:
      _, decoded_outputs, _ = log_decode_results(
          result["inputs"],
          result["outputs"],
          problem_name,
          None,
          inputs_vocab,
          targets_vocab,
          log_results=decode_hp.log_results,
          skip_eos_postprocess=decode_hp.skip_eos_postprocess)
      decodes.append(decoded_outputs)
    total_time_per_step += elapsed_time
    total_cnt += result["outputs"].shape[-1]
  duration = time.time() - start_time
  tf.logging.info("Elapsed Time: %5.5f" % duration)
  tf.logging.info("Averaged Single Token Generation Time: %5.7f "
                  "(time %5.7f count %d)" %
                  (total_time_per_step / total_cnt,
                   total_time_per_step, total_cnt))
  if decode_hp.batch_size == 1:
    tf.logging.info("Inference time %.4f seconds "
                    "(Latency = %.4f ms/setences)" %
                    (duration, 1000.0*duration/num_sentences))
  else:
    tf.logging.info("Inference time %.4f seconds "
                    "(Throughput = %.4f sentences/second)" %
                    (duration, num_sentences/duration))

  # If decode_to_file was provided use it as the output filename without change
  # (except for adding shard_id if using more shards for decoding).
  # Otherwise, use the input filename plus model, hp, problem, beam, alpha.
  decode_filename = decode_to_file if decode_to_file else filename
  if not decode_to_file:
    decode_filename = _decode_filename(decode_filename, problem_name, decode_hp)
  else:
    decode_filename = _add_shard_to_filename(decode_filename, decode_hp)
  tf.logging.info("Writing decodes into %s" % decode_filename)
  outfile = tf.gfile.Open(decode_filename, "w")
  for index in range(len(sorted_inputs)):
    outfile.write("%s%s" % (decodes[sorted_keys[index]], decode_hp.delimiter))
  outfile.flush()
  outfile.close()

  output_dir = os.path.join(estimator.model_dir, "decode")
  tf.gfile.MakeDirs(output_dir)

  run_postdecode_hooks(DecodeHookArgs(
      estimator=estimator,
      problem=hparams.problem,
      output_dirs=[output_dir],
      hparams=hparams,
      decode_hparams=decode_hp,
      predictions=list(result_iter)
  ), None)
930537
def load_source(self):
    """"""
    if self.filename in self.STDIN_NAMES:
      self.filename = 'stdin'
      self.source = pycodestyle.stdin_get_value()
    else:
      with pep257.tokenize_open(self.filename) as fd:
        self.source = fd.read()
746503
def _Decode(self, codec_name, data):
    """"""
    try:
      return data.decode(codec_name, "replace")
    except LookupError:
      raise RuntimeError("Codec could not be found.")
    except AssertionError:
      raise RuntimeError("Codec failed to decode")
1012539
def readme(filename, encoding='utf8'):
    

    with io.open(filename, encoding=encoding) as source:
        return source.read()
380091
def detect_encoding(filename):
    """"""
    try:
        with open(filename, 'rb') as input_file:
            from lib2to3.pgen2 import tokenize as lib2to3_tokenize
            encoding = lib2to3_tokenize.detect_encoding(input_file.readline)[0]

            # Check for correctness of encoding.
            with open_with_encoding(filename, encoding) as input_file:
                input_file.read()

        return encoding
    except (SyntaxError, LookupError, UnicodeDecodeError):
        return 'latin-1'
692492
def guess_file_encoding(fh, default=DEFAULT_ENCODING):
    """"""
    start = fh.tell()
    detector = chardet.UniversalDetector()
    while True:
        data = fh.read(1024 * 10)
        if not data:
            detector.close()
            break
        detector.feed(data)
        if detector.done:
            break

    fh.seek(start)
    return normalize_result(detector.result, default=default)
232350
def _decode(self, infile, encoding):
        
        if isinstance(infile, string_types):
            # can't be unicode
            # NOTE: Could raise a ``UnicodeDecodeError``
            return infile.decode(encoding).splitlines(True)
        for i, line in enumerate(infile):
            # NOTE: The isinstance test here handles mixed lists of unicode/string
            # NOTE: But the decode will break on any non-string values
            # NOTE: Or could raise a ``UnicodeDecodeError``
            if PY3K:
                if not isinstance(line, str):
                    infile[i] = line.decode(encoding)
            else:
                if not isinstance(line, unicode):
                    infile[i] = line.decode(encoding)
        return infile
492083
def _decode(rawstr):
    
    # Check for the magick word.
    try:
        rawstr = rawstr.decode('utf-8')
    except (AttributeError, UnicodeEncodeError):
        pass
    except (UnicodeDecodeError):
        try:
            rawstr = rawstr.decode('iso-8859-1')
        except (UnicodeDecodeError):
            rawstr = rawstr.decode('utf-8', 'ignore')
    if not rawstr.startswith(_MAGICK):
        raise MessageError("This is not a '%s' message (wrong magick word)"
                           % _MAGICK)
    rawstr = rawstr[len(_MAGICK):]

    # Check for element count and version
    raw = re.split(r"\s+", rawstr, maxsplit=6)
    if len(raw) < 5:
        raise MessageError("Could node decode raw string: '%s ...'"
                           % str(rawstr[:36]))
    version = raw[4][:len(_VERSION)]
    if not _is_valid_version(version):
        raise MessageError("Invalid Message version: '%s'" % str(version))

    # Start to build message
    msg = dict((('subject', raw[0].strip()),
                ('type', raw[1].strip()),
                ('sender', raw[2].strip()),
                ('time', strp_isoformat(raw[3].strip())),
                ('version', version)))

    # Data part
    try:
        mimetype = raw[5].lower()
        data = raw[6]
    except IndexError:
        mimetype = None

    if mimetype is None:
        msg['data'] = ''
        msg['binary'] = False
    elif mimetype == 'application/json':
        try:
            msg['data'] = json.loads(raw[6], object_hook=datetime_decoder)
            msg['binary'] = False
        except ValueError:
            raise MessageError("JSON decode failed on '%s ...'" % raw[6][:36])
    elif mimetype == 'text/ascii':
        msg['data'] = str(data)
        msg['binary'] = False
    elif mimetype == 'binary/octet-stream':
        msg['data'] = data
        msg['binary'] = True
    else:
        raise MessageError("Unknown mime-type '%s'" % mimetype)

    return msg
327785
def decode(raw):
    """"""
    return raw \
        .replace(bytes([SLIP_ESC, SLIP_ESC_END]), bytes([SLIP_END])) \
        .replace(bytes([SLIP_ESC, SLIP_ESC_ESC]), bytes([SLIP_ESC]))
801647
def decode_if_bytes(obj, mode=True):
    """"""
    if mode is True:
        mode = unicode_errors_default
    if isinstance(obj, bytes):
        return obj.decode("utf-8", errors=mode)
    return obj