Turn a byte string from the command line into a unicode string.
190097
def decodeCommandLine(self, cmdline):
        
        codec = getattr(sys.stdin, 'encoding', None) or sys.getdefaultencoding()
        return unicode(cmdline, codec)
926743
def to_unicode(obj, encoding='utf-8'):
    """"""
    if isinstance(obj, basestring):
        if not isinstance(obj, unicode):
            obj = unicode(obj, encoding)
    return obj
4213
def to_unicode(data, encoding='UTF-8'):
    """"""
    if isinstance(data, unicode_type):
        return data

    if isinstance(data, bytes):
        return unicode_type(data, encoding=encoding)

    if hasattr(data, '__iter__'):
        try:
            dict(data)
        except TypeError:
            pass
        except ValueError:
            # Assume it's a one dimensional data structure
            return (to_unicode(i, encoding) for i in data)
        else:
            # We support 2.6 which lacks dict comprehensions
            if hasattr(data, 'items'):
                data = data.items()
            return dict(((to_unicode(k, encoding), to_unicode(v, encoding)) for k, v in data))

    return data
1193994
def make_str(value):
    """"""
    if isinstance(value, bytes):
        try:
            return value.decode(get_filesystem_encoding())
        except UnicodeError:
            return value.decode('utf-8', 'replace')
    return text_type(value)
406771
def utf8(value):
    
    """
    try:
        if isinstance(value, bytes):
            return value.decode('utf-8')
        else:
            return value
    except Exception:
        raise ValueError('Not UTF-8: %r' % value)
1108228
def _to_unicode(instr):
    '''
    
    '''
    if instr is None or isinstance(instr, six.text_type):
        return instr
    else:
        return six.text_type(instr, 'utf8')
285689
def to_unicode(value):
    
    if isinstance(value, _TO_UNICODE_TYPES):
        return value
    if not isinstance(value, bytes_type):
        raise TypeError(
            "Expected bytes, unicode, or None; got %r" % type(value)
        )
    return value.decode("utf-8")
941126
def _encode_string(string):
    """"""
    if not isinstance(string, bytes):
        string = string.encode('utf8')
    return ffi.new('char[]', string)
59905
def unicode2ascii(text, expand=True):
    
    """
    translate = UTF8_TO_ASCII if not expand else UTF8_TO_MULTIASCII
    output = ''
    for c in text:
        if not c or ord(c) < 128:
            output += c
        else:
            output += translate[c] if c in translate else ' '
    return output.strip()
657454
def bytes_to_ustr(self, b):
        ""
        return b.decode(charset_map.get(self.charset, self.charset))
297080
def str_to_bytes(self, s):
        ""
        if (PYTHON_MAJOR_VER == 3 or
                (PYTHON_MAJOR_VER == 2 and type(s) == unicode)):
            return s.encode(charset_map.get(self.charset, self.charset))
        return s
1113636
def to_bytes(s):  # pragma: nocover
    
    if isinstance(s, bytes):
        return s
    if isinstance(s, str) or is_unicode(s):
        return s.encode("utf-8")
    try:
        return unicode(s).encode("utf-8")
    except NameError:
        return str(s).encode("utf-8")
1015682
def safe_unicode(string):
    """"""
    if not PY3:
        uni = string.replace(u'\u2019', "'")
        return uni.encode('utf-8')
        
    return string
1262900
def ucnstring_to_unicode(ucn_string):
    """"""
    ucn_string = ucnstring_to_python(ucn_string).decode('utf-8')

    assert isinstance(ucn_string, text_type)
    return ucn_string
1057674
def from_bytes(self, string):
        """"""
        msg = srsly.msgpack_loads(gzip.decompress(string))
        self.attrs = msg["attrs"]
        self.strings = set(msg["strings"])
        lengths = numpy.fromstring(msg["lengths"], dtype="int32")
        flat_spaces = numpy.fromstring(msg["spaces"], dtype=bool)
        flat_tokens = numpy.fromstring(msg["tokens"], dtype="uint64")
        shape = (flat_tokens.size // len(self.attrs), len(self.attrs))
        flat_tokens = flat_tokens.reshape(shape)
        flat_spaces = flat_spaces.reshape((flat_spaces.size, 1))
        self.tokens = NumpyOps().unflatten(flat_tokens, lengths)
        self.spaces = NumpyOps().unflatten(flat_spaces, lengths)
        for tokens in self.tokens:
            assert len(tokens.shape) == 2, tokens.shape
        return self
996804
def from_bytes(self, bytes_representation, resource=None):
        
        text = bytes_representation.decode(self.encoding)
        return self.from_string(text, resource=resource)
44099
def convertascii(value, command='to'):
    
    command = command.lower()
    if command == 'to':
        return chr(value)
    elif command == 'from':
        return ord(value)
    else:
        raise ValueError('Invalid operation provided.')