Internal function, deals with different Python versions, unicode strings versus bytes, and with the leak bug in lxml
199127
def xmltreefromstring(s):
    """"""
    if sys.version < '3':
        #Python 2
        if isinstance(s,unicode): #pylint: disable=undefined-variable
            s = s.encode('utf-8')
        try:
            return ElementTree.parse(StringIO(s), ElementTree.XMLParser(collect_ids=False))
        except TypeError:
            return ElementTree.parse(StringIO(s), ElementTree.XMLParser()) #older lxml, may leak!!!!
    else:
        #Python 3
        if isinstance(s,str):
            s = s.encode('utf-8')
        try:
            return ElementTree.parse(BytesIO(s), ElementTree.XMLParser(collect_ids=False))
        except TypeError:
            return ElementTree.parse(BytesIO(s), ElementTree.XMLParser())
239402
def __xinclude_lxml(target, source, env):
    
    from lxml import etree
    
    doc = etree.parse(str(source[0]))
    doc.xinclude()
    try:
        doc.write(str(target[0]), xml_declaration=True, 
                  encoding="UTF-8", pretty_print=True)
    except:
        pass

    return None
227646
def _matching_string(matched, string):
        
        if string is None:
            return string
        if IS_PY2:
            # pylint: disable=undefined-variable
            if isinstance(matched, text_type):
                return text_type(string)
        else:
            if isinstance(matched, bytes) and isinstance(string, str):
                return string.encode(locale.getpreferredencoding(False))
        return string
1088105
def get_encodings_from_content(content):
    

    charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
    pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
    xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')

    return (charset_re.findall(content) +
            pragma_re.findall(content) +
            xml_re.findall(content))
337622
def encode_unicode(f):
    
    @wraps(f)
    def wrapped(obj, error):

        def _encode(value):
            """Helper encoding unicode strings into binary utf-8"""
            if isinstance(value, unicode):  # noqa: F821
                return value.encode('utf-8')
            return value

        error = copy(error)
        error.document_path = _encode(error.document_path)
        error.schema_path = _encode(error.schema_path)
        error.constraint = _encode(error.constraint)
        error.value = _encode(error.value)
        error.info = _encode(error.info)
        return f(obj, error)

    return wrapped if PYTHON_VERSION < 3 else f
773060
def force_unicode(s, encoding='utf-8', errors='strict'):
    
    # Handle the common case first, saves 30-40% when s is an instance of
    # six.text_type. This function gets called often in that setting.
    if isinstance(s, six.text_type):
        return s
    if not isinstance(s, six.string_types):
        if six.PY3:
            if isinstance(s, bytes):
                s = six.text_type(s, encoding, errors)
            else:
                s = six.text_type(s)
        else:
            s = six.text_type(bytes(s), encoding, errors)
    else:
        # Note: We use .decode() here, instead of six.text_type(s,
        # encoding, errors), so that if s is a SafeBytes, it ends up being
        # a SafeText at the end.
        s = s.decode(encoding, errors)
    return s
87369
def unicode_convert(obj):
    
    try:
        if isinstance(obj, dict):
            return {unicode_convert(key): unicode_convert(value) for key, value in obj.items()}
        elif isinstance(obj, list):
            return [unicode_convert(element) for element in obj]
        elif isinstance(obj, str):
            return obj 
        elif isinstance(obj, six.text_type):
            return obj.encode('utf-8')
        elif isinstance(obj, six.integer_types):
            return obj
        else:
            return obj
    except:
        return obj
100804
def ignore_unicode_prefix(f):
    
    if sys.version >= '3':
        # the representation of unicode string in Python 3 does not have prefix 'u',
        # so remove the prefix 'u' for doc tests
        literal_re = re.compile(r"(\W|^)[uU](['])", re.UNICODE)
        f.__doc__ = literal_re.sub(r'\1\2', f.__doc__)
    return f
372449
def force_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
    
    # Handle the common case first for performance reasons.
    if isinstance(s, bytes):
        if encoding == 'utf-8':
            return s
        else:
            return s.decode('utf-8', errors).encode(encoding, errors)
    if strings_only and is_protected_type(s):
        return s
    if isinstance(s, memoryview):
        return bytes(s)
    if not isinstance(s, str):
        return str(s).encode(encoding, errors)
    else:
        return s.encode(encoding, errors)
215031
def safe_unicode(value):
    
    if sys.version_info < (3,0):
        if isinstance(value, str):
            return value.decode('utf-8')
        else:
            return unicode(value)
    else:
        return str(value)
720770
def ensure_unicode(str_):
    
    if isinstance(str_, __STR__):
        return str_
    else:
        try:
            return __STR__(str_)
        except UnicodeDecodeError:
            if str_.startswith(codecs.BOM_UTF8):
                # Can safely remove the utf8 marker
                # http://stackoverflow.com/questions/12561063/python-extract-data-from-file
                str_ = str_[len(codecs.BOM_UTF8):]
            return str_.decode('utf-8')
624315
def _ensure_unicode_string(string):
        
        if not isinstance(string, six.text_type):
            string = string.decode('utf-8')
        return string
341664
def _unicode(p):
    
    q = {}
    for tag in p:
        vals = []
        for v in p[tag]:
            if type(v) is not str:                                                              # pragma: no cover
                v = v.decode('utf-8')
            vals.append(v)
        if type(tag) is not str:                                                                # pragma: no cover
            tag = tag.decode('utf-8')
        q[tag] = vals
    return q
72607
def _RemoveIllegalXMLCharacters(self, xml_string):
    
    if not isinstance(xml_string, py2to3.STRING_TYPES):
      return xml_string

    return self._ILLEGAL_XML_RE.sub('\ufffd', xml_string)
146817
def obj2unicode(obj):
    
    if isinstance(obj, unicode_type):
        return obj
    elif isinstance(obj, bytes_type):
        try:
            return unicode_type(obj, 'utf-8')
        except UnicodeDecodeError as strerror:
            sys.stderr.write("UnicodeDecodeError exception for string '%s': %s\n" % (obj, strerror))
            return unicode_type(obj, 'utf-8', 'replace')
    else:
        return unicode_type(obj)
1093692
def parseXRDS(text):
    
    try:
        # lxml prefers to parse bytestrings, and occasionally chokes on a
        # combination of text strings and declared XML encodings -- see
        # https://github.com/necaris/python3-openid/issues/19
        # To avoid this, we ensure that the 'text' we're parsing is actually
        # a bytestring
        bytestring = text.encode('utf8') if isinstance(text, str) else text
        element = SafeElementTree.XML(bytestring)
    except (SystemExit, MemoryError, AssertionError, ImportError):
        raise
    except Exception as why:
        exc = XRDSError('Error parsing document as XML')
        exc.reason = why
        raise exc
    else:
        tree = ElementTree.ElementTree(element)
        if not isXRDS(tree):
            raise XRDSError('Not an XRDS document')

        return tree
801398
def try_utf8_decode(value):
    
    if not value or not is_string(value):
        return value
    elif PYTHON3 and not isinstance(value, bytes):
        return value
    elif not PYTHON3 and not isinstance(value, unicode):
        return value

    try:
        return value.decode('utf-8')
    except UnicodeDecodeError:
        pass

    return value