Decode data coming from the database if needed and convert rows to tuples
        (pyodbc Rows are not sliceable).
488163
def format_results(self, rows):
        
        needs_utc = _DJANGO_VERSION >= 14 and settings.USE_TZ
        if not (needs_utc or not self.driver_supports_utf8):
            return tuple(rows)
        # FreeTDS (and other ODBC drivers?) don't support Unicode yet, so we
        # need to decode UTF-8 data coming from the DB
        fr = []
        for row in rows:
            if not self.driver_supports_utf8 and isinstance(row, binary_type):
                row = row.decode(self.encoding)

            elif needs_utc and isinstance(row, datetime.datetime):
                row = row.replace(tzinfo=timezone.utc)
            fr.append(row)
        return tuple(fr)
423143
def decode(self, encoding=None):
        '''  '''
        copy = FormsDict()
        enc = copy.input_encoding = encoding or self.input_encoding
        copy.recode_unicode = False
        for key, value in self.allitems():
            copy.append(self._fix(key, enc), self._fix(value, enc))
        return copy
29045
def _try_decode_dict(self, content):
        
        result = dict()
        for key, value in content.items():
            key = try_utf8_decode(key)
            if isinstance(value, dict):
                result[key] = self._try_decode_dict(value)
            elif isinstance(value, list):
                result[key] = self._try_decode_list(value)
            elif isinstance(value, tuple):
                result[key] = self._try_decode_tuple(value)
            else:
                result[key] = try_utf8_decode(value)
        return result
216722
def from_df(data_frame):
        
        labels = data_frame.keys().tolist()
        data = data_frame.values.tolist()
        return SqlTable(labels, data, "{:.3f}", "\n")
1050116
def _decode(self, s, encode_nominal=False, matrix_type=DENSE):
        ''''''

        # Make sure this method is idempotent
        self._current_line = 0

        # If string, convert to a list of lines
        if isinstance(s, basestring):
            s = s.strip('\r\n ').replace('\r\n', '\n').split('\n')

        # Create the return object
        obj = {
            u'description': u'',
            u'relation': u'',
            u'attributes': [],
            u'data': []
        }
        attribute_names = {}

        # Create the data helper object
        data = _get_data_object_for_decoding(matrix_type)

        # Read all lines
        STATE = _TK_DESCRIPTION
        s = iter(s)
        for row in s:
            self._current_line += 1
            # Ignore empty lines
            row = row.strip(' \r\n')
            if not row: continue

            u_row = row.upper()

            # DESCRIPTION -----------------------------------------------------
            if u_row.startswith(_TK_DESCRIPTION) and STATE == _TK_DESCRIPTION:
                obj['description'] += self._decode_comment(row) + '\n'
            # -----------------------------------------------------------------

            # RELATION --------------------------------------------------------
            elif u_row.startswith(_TK_RELATION):
                if STATE != _TK_DESCRIPTION:
                    raise BadLayout()

                STATE = _TK_RELATION
                obj['relation'] = self._decode_relation(row)
            # -----------------------------------------------------------------

            # ATTRIBUTE -------------------------------------------------------
            elif u_row.startswith(_TK_ATTRIBUTE):
                if STATE != _TK_RELATION and STATE != _TK_ATTRIBUTE:
                    raise BadLayout()

                STATE = _TK_ATTRIBUTE

                attr = self._decode_attribute(row)
                if attr[0] in attribute_names:
                    raise BadAttributeName(attr[0], attribute_names[attr[0]])
                else:
                    attribute_names[attr[0]] = self._current_line
                obj['attributes'].append(attr)

                if isinstance(attr[1], (list, tuple)):
                    if encode_nominal:
                        conversor = EncodedNominalConversor(attr[1])
                    else:
                        conversor = NominalConversor(attr[1])
                else:
                    CONVERSOR_MAP = {'STRING': unicode,
                                     'INTEGER': lambda x: int(float(x)),
                                     'NUMERIC': float,
                                     'REAL': float}
                    conversor = CONVERSOR_MAP[attr[1]]

                self._conversors.append(conversor)
            # -----------------------------------------------------------------

            # DATA ------------------------------------------------------------
            elif u_row.startswith(_TK_DATA):
                if STATE != _TK_ATTRIBUTE:
                    raise BadLayout()

                break
            # -----------------------------------------------------------------

            # COMMENT ---------------------------------------------------------
            elif u_row.startswith(_TK_COMMENT):
                pass
            # -----------------------------------------------------------------
        else:
            # Never found @DATA
            raise BadLayout()

        def stream():
            for row in s:
                self._current_line += 1
                row = row.strip()
                # Ignore empty lines and comment lines.
                if row and not row.startswith(_TK_COMMENT):
                    yield row

        # Alter the data object
        obj['data'] = data.decode_rows(stream(), self._conversors)
        if obj['description'].endswith('\n'):
            obj['description'] = obj['description'][:-1]

        return obj
212109
def decode_bytes(array):
    
    if (sys.version_info.major == 2 or not len(array) or
        (isinstance(array, np.ndarray) and array.dtype.kind != 'O')):
        return array
    decoded = [v.decode('utf-8') if isinstance(v, bytes) else v for v in array]
    if isinstance(array, np.ndarray):
        return np.asarray(decoded)
    elif isinstance(array, tuple):
        return tuple(decoded)
    return decoded
654848
def decode_string(data, encoding='hex'):
    '''
    
    '''
    if six.PY2:
        return data.decode(encoding)
    else:
        return codecs.decode(data.encode('ascii'), encoding)
1263417
def to_named_tuple(keys, values):
    

    values = [dbobject.__dict__[key] for key in dbobject.keys()]

    tuple_object = namedtuple('DBObject', dbobject.keys())

    tuple_instance = tuple_object._make(values)

    return tuple_instance
547301
def decode_dict(data):
    
    if six.PY3:
        return data
    rv = {}
    for key, value in iteritems(data):
        if isinstance(key, text_type):
            key = key.encode('utf-8')

        if isinstance(value, text_type):
            value = value.encode('utf-8')
        elif isinstance(value, list):
            value = _decode_list(value)
        # no need to recurse into dict, json library will do that
        rv[key] = value
    return rv
913479
def decode_sql(self, sql):
        
        # JSON is defined as using "unicode", we'll go a step further and
        # mandate utf-8 (though for the base64 part, it doesn't really matter!)
        base64_sql_bytes = to_unicode(sql).encode('utf-8')
        # in python3.x you can pass `validate=True` to b64decode to get this
        # behavior.
        if not re.match(b'^[A-Za-z0-9+/]*={0,2}$', base64_sql_bytes):
            self.raise_invalid_base64(sql)

        try:
            sql_bytes = base64.b64decode(base64_sql_bytes)
        except ValueError:
            self.raise_invalid_base64(sql)

        return sql_bytes.decode('utf-8')
763755
def decode_timeseries_row(self, tsrow, tscols=None,
                              convert_timestamp=False):
        
        row = []
        for i, cell in enumerate(tsrow.cells):
            col = None
            if tscols is not None:
                col = tscols[i]
            if cell.HasField('varchar_value'):
                if col and not (col.type == TsColumnType.Value('VARCHAR') or
                                col.type == TsColumnType.Value('BLOB')):
                    raise TypeError('expected VARCHAR or BLOB column')
                else:
                    row.append(cell.varchar_value)
            elif cell.HasField('sint64_value'):
                if col and col.type != TsColumnType.Value('SINT64'):
                    raise TypeError('expected SINT64 column')
                else:
                    row.append(cell.sint64_value)
            elif cell.HasField('double_value'):
                if col and col.type != TsColumnType.Value('DOUBLE'):
                    raise TypeError('expected DOUBLE column')
                else:
                    row.append(cell.double_value)
            elif cell.HasField('timestamp_value'):
                if col and col.type != TsColumnType.Value('TIMESTAMP'):
                    raise TypeError('expected TIMESTAMP column')
                else:
                    dt = cell.timestamp_value
                    if convert_timestamp:
                        dt = datetime_from_unix_time_millis(
                            cell.timestamp_value)
                    row.append(dt)
            elif cell.HasField('boolean_value'):
                if col and col.type != TsColumnType.Value('BOOLEAN'):
                    raise TypeError('expected BOOLEAN column')
                else:
                    row.append(cell.boolean_value)
            else:
                row.append(None)
        return row
196629
def decode_response(client_message, to_object=None):
    
    parameters = dict(read_count=None, items=None)
    parameters['read_count'] = client_message.read_int()
    items_size = client_message.read_int()
    items = []
    for _ in range(0, items_size):
        items_item = client_message.read_data()
        items.append(items_item)
    parameters['items'] = ImmutableLazyDataList(items, to_object)
    return parameters
1255401
def decode_response(data):
    '''
    '''
    res = CaseInsensitiveDict()
    for dataline in data.decode('utf-8').splitlines()[1:]:
        dataline = dataline.strip()
        if not dataline:
            continue
        line_parts = dataline.split(':', 1)
        # This is to deal with headers with no value.
        if len(line_parts) < 2:
            line_parts = (line_parts[0], '')
        res[line_parts[0].strip()] = line_parts[1].strip()
    return res
299757
def _sql_params(sql):
    
    params = None
    if isinstance(sql, (list, tuple)):
        elements = len(sql)
        if elements == 2:
            sql, params = sql
        else:
            raise ValueError("Expected a 2-tuple but got %d" % elements)
    return sql, params
1190277
def record_to_objects(self):
        """"""
        from ambry.orm.exc import NotFoundError

        fr = self.record

        contents = fr.unpacked_contents

        if not contents:
            return

        # Zip transposes an array when in the form of a list of lists, so this transposes so
        # each row starts with the heading and the rest of the row are the values
        # for that row. The bool and filter return false when none of the values
        # are non-empty. Then zip again to transpose to original form.

        non_empty_rows = drop_empty(contents)

        s = self._dataset._database.session

        for i, row in enumerate(non_empty_rows):

            if i == 0:
                header = row
            else:
                d = dict(six.moves.zip(header, row))

                if 'widths' in d:
                    del d['widths']  # Obsolete column in old spreadsheets.

                if 'table' in d:
                    d['dest_table_name'] = d['table']
                    del d['table']

                if 'order' in d:
                    d['stage'] = d['order']
                    del d['order']

                if 'dest_table' in d:
                    d['dest_table_name'] = d['dest_table']
                    del d['dest_table']

                if 'source_table' in d:
                    d['source_table_name'] = d['source_table']
                    del d['source_table']

                d['d_vid'] = self._dataset.vid

                d['state'] = 'synced'

                try:
                    ds = self._dataset.source_file(str(d['name']))
                    ds.update(**d)
                except NotFoundError:
                    name = d['name']
                    del d['name']

                    try:
                        ds = self._dataset.new_source(name, **d)
                    except:
                        print(name, d)
                        import pprint
                        pprint.pprint(d)
                        raise
                except:  # Odd error with 'none' in keys for d
                    print('!!!', header)
                    print('!!!', row)
                    raise

                s.merge(ds)

        self._dataset._database.commit()
804891
def _to_df(self, query, conn, index_col=None, coerce_float=True, params=None,
               parse_dates=None, columns=None):
        
        return pd.io.sql.read_sql(str(query), conn, index_col=index_col,
                                  coerce_float=coerce_float, params=params,
                                  parse_dates=parse_dates, columns=columns)
302082
def from_dict(cls, database, key, data, clear=False):
        
        zset = cls(database, key)
        if clear:
            zset.clear()
        zset.add(data)
        return zset