Converts input to unicode if necessary.
    If `s` is bytes, it will be decoded using the `encoding` parameters.
    This function is used for preprocessing /source/ and /filename/ arguments
    to the builtin function `compile`.
495562
def ensureUtf(s, encoding='utf8'):
    
    # In Python2, str == bytes.
    # In Python3, bytes remains unchanged, but str means unicode
    # while unicode is not defined anymore
    if type(s) == bytes:
        return s.decode(encoding, 'ignore')
    else:
        return s
1330202
def stdout_encode(u, default='utf-8'):
    
    # from http://stackoverflow.com/questions/3627793/best-output-type-and-
    #   encoding-practices-for-repr-functions
    encoding = sys.stdout.encoding or default
    return u.encode(encoding, "replace").decode(encoding, "replace")
926133
def make_unicode(string):
    
    if sys.version < '3' and isinstance(string, str):
        return unicode(string.decode('utf-8'))

    return string
252200
def to_unicode(s):
    
    if not isinstance(s, TEXT):
        if not isinstance(s, bytes):
            raise TypeError('You are required to pass either unicode or '
                            'bytes here, not: %r (%s)' % (type(s), s))
        try:
            s = s.decode('utf-8')
        except UnicodeDecodeError as le:
            raise TypeError('You are required to pass either a unicode '
                            'object or a utf-8-encoded bytes string here. '
                            'You passed a bytes object which contained '
                            'non-utf-8: %r. The UnicodeDecodeError that '
                            'resulted from attempting to interpret it as '
                            'utf-8 was: %s'
                            % (s, le,))
    return s
1345183
def native_string_to_bytes(s, encoding="ascii", errors="strict"):
    
    if not isinstance(s, str):
        raise TypeError("{} must be type str, not {}".format(s, type(s)))
    if str is bytes:
        # Python 2
        return s
    else:
        # Python 3
        return s.encode(encoding=encoding, errors=errors)
1036887
def parse_source(self, bytessrc, compile_info):
        
        # Detect source encoding.
        enc = None
        if compile_info.flags & consts.PyCF_SOURCE_IS_UTF8:
            enc = 'utf-8'

        if compile_info.flags & consts.PyCF_IGNORE_COOKIE:
            textsrc = bytessrc
        elif bytessrc.startswith(b"\xEF\xBB\xBF"):
            bytessrc = bytessrc[3:]
            enc = 'utf-8'
            # If an encoding is explicitly given check that it is utf-8.
            decl_enc = _check_for_encoding(bytessrc)
            if decl_enc and decl_enc != "utf-8":
                raise error.SyntaxError("UTF-8 BOM with %s coding cookie" % decl_enc,
                                        filename=compile_info.filename)
            textsrc = bytessrc.decode('utf-8')
        else:
            enc = _normalize_encoding(_check_for_encoding(bytessrc))
            if enc is None:
                enc = 'utf-8'
            try:
                # textsrc = recode_to_utf8(self.space, bytessrc, enc)
                textsrc = bytessrc.decode(enc)
            except UnicodeDecodeError as e:
                # if the codec is not found, LookupError is raised.  we
                # check using 'is_w' not to mask potential IndexError or
                # KeyError
                space = self.space
                if e.match(space, space.w_LookupError):
                    raise error.SyntaxError("Unknown encoding: %s" % enc,
                                            filename=compile_info.filename)
                # Transform unicode errors into SyntaxError
                if e.match(space, space.w_UnicodeDecodeError):
                    e.normalize_exception(space)
                    w_message = space.str(e.get_w_value(space))
                    raise error.SyntaxError(space.text_w(w_message))
                raise

        flags = compile_info.flags

        # The tokenizer is very picky about how it wants its input.
        source_lines = textsrc.splitlines(True)
        if source_lines and not source_lines[-1].endswith("\n"):
            source_lines[-1] += '\n'
        if textsrc and textsrc[-1] == "\n":
            flags &= ~consts.PyCF_DONT_IMPLY_DEDENT

        self.prepare(_targets[compile_info.mode])
        tp = 0
        try:
            try:
                # Note: we no longer pass the CO_FUTURE_* to the tokenizer,
                # which is expected to work independently of them.  It's
                # certainly the case for all futures in Python <= 2.7.
                tokens = pytokenizer.generate_tokens(source_lines, flags)

                newflags, last_future_import = (
                    future.add_future_flags(self.future_flags, tokens))
                compile_info.last_future_import = last_future_import
                compile_info.flags |= newflags
                self.grammar = pygram.python_grammar
                tokens_stream = iter(tokens)

                for tp, value, lineno, column, line in tokens_stream:
                    if self.add_token(tp, value, lineno, column, line):
                        break

                if compile_info.mode == 'single':
                    for tp, value, lineno, column, line in tokens_stream:
                        if tp == pygram.tokens.ENDMARKER:
                            break
                        if tp == pygram.tokens.NEWLINE:
                            continue

                        if tp == pygram.tokens.COMMENT:
                            for tp, _, _, _, _ in tokens_stream:
                                if tp == pygram.tokens.NEWLINE:
                                    break
                        else:
                            new_err = error.SyntaxError
                            msg = ("multiple statements found while "
                                   "compiling a single statement")
                            raise new_err(msg, lineno, column,
                                          line, compile_info.filename)

            except error.TokenError as e:
                e.filename = compile_info.filename
                raise
            except error.TokenIndentationError as e:
                e.filename = compile_info.filename
                raise
            except parser.ParseError as e:
                # Catch parse errors, pretty them up and reraise them as a
                # SyntaxError.
                new_err = error.IndentationError
                if tp == pygram.tokens.INDENT:
                    msg = "unexpected indent"
                elif e.expected == pygram.tokens.INDENT:
                    msg = "expected an indented block"
                else:
                    new_err = error.SyntaxError
                    msg = "invalid syntax"
                raise new_err(msg, e.lineno, e.col_offset, e.line,
                              compile_info.filename)
            else:
                tree = self.root
        finally:
            # Avoid hanging onto the tree.
            self.root = None
        if enc is not None:
            compile_info.encoding = enc
        return tree
7622
def _to_unicode(self, data, encoding, errors='strict'):
    ''''''

    # strip Byte Order Mark (if present)
    if (len(data) >= 4) and (data[:2] == '\xfe\xff') and (data[2:4] != '\x00\x00'):
        encoding = 'utf-16be'
        data = data[2:]
    elif (len(data) >= 4) and (data[:2] == '\xff\xfe') and (data[2:4] != '\x00\x00'):
        encoding = 'utf-16le'
        data = data[2:]
    elif data[:3] == '\xef\xbb\xbf':
        encoding = 'utf-8'
        data = data[3:]
    elif data[:4] == '\x00\x00\xfe\xff':
        encoding = 'utf-32be'
        data = data[4:]
    elif data[:4] == '\xff\xfe\x00\x00':
        encoding = 'utf-32le'
        data = data[4:]
    newdata = unicode(data, encoding, errors)
    return newdata
1090032
def to_string(s, encoding='utf-8'):
    
    if six.PY2:
        return s.encode(encoding)
    if isinstance(s, bytes):
        return s.decode(encoding)
    return s
337622
def encode_unicode(f):
    
    @wraps(f)
    def wrapped(obj, error):

        def _encode(value):
            """Helper encoding unicode strings into binary utf-8"""
            if isinstance(value, unicode):  # noqa: F821
                return value.encode('utf-8')
            return value

        error = copy(error)
        error.document_path = _encode(error.document_path)
        error.schema_path = _encode(error.schema_path)
        error.constraint = _encode(error.constraint)
        error.value = _encode(error.value)
        error.info = _encode(error.info)
        return f(obj, error)

    return wrapped if PYTHON_VERSION < 3 else f
451008
def to_bytes(s):
    
    if bytes != str:
        if type(s) == str:
            return s.encode('utf-8')
    return s
1058309
def convert_to_unicode( tscii_input ):
    
    output = list()
    prev = None
    prev2x = None
    # need a look ahead of 2 tokens atleast
    for char in tscii_input:
        ## print "%2x"%ord(char) # debugging
        if ord(char) < 128 :
            # base-ASCII copy to output
            output.append( char )
            prev = None
            prev2x = None
            
        elif ord(char) in TSCII_DIRECT_LOOKUP:
            if ( prev in TSCII_PRE_MODIFIER ):
                curr_char = [TSCII[ord(char)],TSCII[prev]]                
            else:
                # we are direct lookup char
                curr_char = [TSCII[ord(char)]]
                char = None

            output.extend( curr_char )

        elif ( (ord(char) in TSCII_POST_MODIFIER) ): 
            
            if ( (prev in TSCII_DIRECT_LOOKUP) and 
                 (prev2x in TSCII_PRE_MODIFIER) ):
                if len(output) >= 2:
                    del output[-1] #we are reducing this token to something new
                    del output[-2]
                elif len(output)==1:
                    del output[-1] 
                else:
                    # nothing to delete here.. 
                    pass
                output.extend( [TSCII[prev], TSCII[prev2x]] )
            else:
                print("Warning: malformed TSCII encoded file; skipping characters")
            
            prev = None
            char = None
        else:
            # pass - must be one of the pre/post modifiers
            pass
        
        prev2x = prev
        if char:
            prev = ord(char)
    return u"".join(output)
1265375
def decode(s):
    
    if isinstance(s, bytes):
        return s.decode(sys.getdefaultencoding())
    else:
        return s
891205
def encode_basestring(s, _PY3=PY3, _q=u('"')):
    
    if _PY3:
        if isinstance(s, binary_type):
            s = s.decode('utf-8')
    else:
        if isinstance(s, str) and HAS_UTF8.search(s) is not None:
            s = s.decode('utf-8')
    def replace(match):
        return ESCAPE_DCT[match.group(0)]
    return _q + ESCAPE.sub(replace, s) + _q
278974
def convert_unicode(value):
    
    if isinstance(value, dict):
        return {convert_unicode(key): convert_unicode(value)
                for key, value in value.iteritems()}
    elif isinstance(value, list):
        return [convert_unicode(item) for item in value]
    elif isinstance(value, unicode):
        return value.encode('utf-8')
    else:
        return value
943194
def mb_code(s, coding=None, errors='replace'):
    """"""
    if isinstance(s, unicode):
        return s if coding is None else s.encode(coding, errors=errors)
    for c in ('utf-8', 'gb2312', 'gbk', 'gb18030', 'big5'):
        try:
            s = s.decode(c)
            return s if coding is None else s.encode(coding, errors=errors)
        except:
            pass
    return unicode(s, errors=errors)
663959
def compile(node,
                uri,
                filename=None,
                default_filters=None,
                buffer_filters=None,
                imports=None,
                future_imports=None,
                source_encoding=None,
                generate_magic_comment=True,
                disable_unicode=False,
                strict_undefined=False,
                enable_loop=True,
                reserved_names=frozenset()):

    """"""

    # if on Py2K, push the "source_encoding" string to be
    # a bytestring itself, as we will be embedding it into
    # the generated source and we don't want to coerce the
    # result into a unicode object, in "disable_unicode" mode
    if not compat.py3k and isinstance(source_encoding, compat.text_type):
        source_encoding = source_encoding.encode(source_encoding)


    buf = util.FastEncodingBuffer()

    printer = PythonPrinter(buf)
    _GenerateRenderMethod(printer,
                            _CompileContext(uri,
                                            filename,
                                            default_filters,
                                            buffer_filters,
                                            imports,
                                            future_imports,
                                            source_encoding,
                                            generate_magic_comment,
                                            disable_unicode,
                                            strict_undefined,
                                            enable_loop,
                                            reserved_names),
                                node)
    return buf.getvalue()
212507
def __needs_encoding(self, s):
        
        if isinstance(s, basestring):
            for c in self.special:
                if c in s:
                    return True