Decode a Pymongo SON object into an Pandas DataFrame
482126
def objify(self, doc, columns=None):
        
        cols = columns or doc[METADATA][COLUMNS]
        data = {}

        for col in cols:
            # if there is missing data in a chunk, we can default to NaN
            # and pandas will autofill the missing values to the correct length
            if col not in doc[METADATA][LENGTHS]:
                d = [np.nan]
            else:
                d = decompress(doc[DATA][doc[METADATA][LENGTHS][col][0]: doc[METADATA][LENGTHS][col][1] + 1])
                # d is ready-only but that's not an issue since DataFrame will copy the data anyway.
                d = np.frombuffer(d, doc[METADATA][DTYPE][col])

                if MASK in doc[METADATA] and col in doc[METADATA][MASK]:
                    mask_data = decompress(doc[METADATA][MASK][col])
                    mask = np.frombuffer(mask_data, 'bool')
                    d = ma.masked_array(d, mask)
            data[col] = d

        # Copy into
        return pd.DataFrame(data, columns=cols, copy=True)[cols]
1032991
def as_format(item, format_str='.2f'):
    
    if isinstance(item, pd.Series):
        return item.map(lambda x: format(x, format_str))
    elif isinstance(item, pd.DataFrame):
        return item.applymap(lambda x: format(x, format_str))
763143
def to_df(obj, cfg={}, raise_on_error=True, silent=False, verbose=False):
    
    
    if type(obj) == str:
        name = obj
    else:
        name = obj.name
    name = name[name.rfind('/') + 1:]
    
    if not raise_on_error:
        try:
            return to_df(obj=obj, cfg=cfg, raise_on_error=True)
        except (pd.parser.CParserError, AttributeError, ValueError, TypeError, IOError) as e:
            if not silent:
                print('WARNING in {}: {} {}'.format(name, e.__class__, e))
            return None
        except:
            if not silent:
                print('WARNING in {}: {}'.format(name, sys.exc_info()[0]))
            return None

    params = {}
    if 'default' in cfg:
        params = deepcopy(cfg['default'])
    if name in cfg:
        for e in cfg[name]:
            params[e] = deepcopy(cfg[name][e])
    if 'custom_date_parser' in params:
        params['date_parser'] = DtParser(params['custom_date_parser']).parse
        del params['custom_date_parser']

    if verbose:
        print(name, params)
    
    if name.lower().startswith('s3:'):
        if not find_spec('boto'):
            raise ImportError('reading from aws-s3 requires the boto package to be installed')
    
    if '.csv' in name.lower():
        # name can be .csv.gz or .csv.bz2
        return pd.read_csv(obj, **params)
        
    elif '.tsv' in name.lower() or '.txt' in name.lower():
        # name can be .tsv.gz or .txt.bz2
        return pd.read_table(obj, **params)
    
    elif name.lower().endswith(('.htm', '.html')):
        if not find_spec('lxml'):
            params['flavor'] = 'bs4'
            if not find_spec('bs4') and not find_spec('html5lib'):
                raise ImportError('reading html requires the lxml or bs4 + html5lib packages to be installed')

        if 'nrows' in params:
            del params['nrows']
        
        if type(obj) == zipfile.ZipExtFile:
            obj = obj.read()
        data = pd.read_html(obj, **params)
        data = {str(i): data[i] for i in range(len(data))}
        return data
    
    elif name.lower().endswith('.xml'):
        if 'nrows' in params:
            del params['nrows']
        
        from icy.utils import xml_to_json
        
        with open(obj) as f:
            json = xml_to_json(str(f.read()))
        
        return pd.read_json(json, **params)
        
    elif name.lower().endswith('.json'):
        if 'nrows' in params:
            del params['nrows']
        
        return pd.read_json(obj, **params)
    
    elif name.lower().endswith(('.xls', '.xlsx')):
        if not find_spec('xlrd'):
            raise ImportError('reading excel files requires the xlrd package to be installed')
        
        if 'nrows' in params:
            del params['nrows']
        
        data = {}
        xls = pd.ExcelFile(obj)
        for key in xls.sheet_names:
            data[key] = xls.parse(key, **params)
        return data
    
    elif name.lower().endswith(('.h5', '.hdf5')):
        if not find_spec('tables'):
            raise ImportError('reading hdf5 files requires the pytables package to be installed')
        
        if 'nrows' in params:
            del params['nrows']
            # params['chunksize'] = params.pop('nrows') # returns iterator
        
        with pd.HDFStore(obj) as store:
            data = {}
            for key in store.keys():
                data[key[1:]] = store[key]
        return data
    
    elif name.lower().endswith(('.sqlite', '.sql', '.db')):
        import sqlite3
        if type(obj) != str:
            raise IOError('sqlite-database must be decompressed before import')
        
        if 'nrows' in params:
            del params['nrows']
            # params['chunksize'] = params.pop('nrows') # returns iterator
        
        with sqlite3.connect(obj) as con:
            data = {}
            cursor = con.cursor()
            cursor.execute('SELECT name FROM sqlite_master WHERE type="table"')
            tables = [t[0] for t in cursor.fetchall()]
            for t in tables:
                sql = 'SELECT * FROM ' + t
                data[t] = pd.read_sql_query(sql, con, **params)
        return data
    
    else:
        try:
            data = {name: odo(obj, pd.DataFrame)}
            if type(data[name]) == pd.DataFrame:
                return data
        except NotImplementedError:
            pass
        raise NotImplementedError('Error creating DataFrame from object', obj)
1050437
def _get_object(data, position, obj_end, opts, dummy):
    """"""
    obj_size = _UNPACK_INT(data[position:position + 4])[0]
    end = position + obj_size - 1
    if data[end:position + obj_size] != b"\x00":
        raise InvalidBSON("bad eoo")
    if end >= obj_end:
        raise InvalidBSON("invalid object length")
    if _raw_document_class(opts.document_class):
        return (opts.document_class(data[position:end + 1], opts),
                position + obj_size)

    obj = _elements_to_dict(data, position + 4, end, opts)

    position += obj_size
    if "$ref" in obj:
        return (DBRef(obj.pop("$ref"), obj.pop("$id", None),
                      obj.pop("$db", None), obj), position)
    return obj, position
137939
def decode(self, dynamo_value):
        
        type, value = next(six.iteritems(dynamo_value))
        if type == STRING:
            return value
        elif type == BINARY:
            return Binary(value)
        elif type == NUMBER:
            return Decimal(value)
        elif type == STRING_SET:
            return set(value)
        elif type == BINARY_SET:
            return set((Binary(v) for v in value))
        elif type == NUMBER_SET:
            return set((Decimal(v) for v in value))
        elif type == BOOL:
            return value
        elif type == LIST:
            return [self.decode(v) for v in value]
        elif type == MAP:
            decoded_dict = {}
            for k, v in six.iteritems(value):
                decoded_dict[k] = self.decode(v)
            return decoded_dict
        elif type == NULL:
            return None
        else:
            raise TypeError("Received unrecognized type %r from dynamo", type)
1140669
def from_bytes(self, raw):
        ''''''
        if len(raw) < 4:
            raise NotEnoughDataError("Not enough bytes ({}) to reconstruct a Null object".format(len(raw)))
        fields = struct.unpack('=I', raw[:4])
        self._af = fields[0]
        return raw[4:]
918043
def transform_incoming(self, son, collection):
        
        for (key, value) in son.items():
            if isinstance(value, dict):  # Make sure we recurse into sub-docs
                son[key] = self.transform_incoming(value, collection)
            elif hasattr(value, 'serialize'):
                LOG.debug("Serializing object: %s", value)
                son[key] = value = value.serialize()
        return son
958030
def _bson_to_dict(data, opts):
    """"""
    try:
        if _raw_document_class(opts.document_class):
            return opts.document_class(data, opts)
        _, end = _get_object_size(data, 0, len(data))
        return _elements_to_dict(data, 4, end, opts)
    except InvalidBSON:
        raise
    except Exception:
        # Change exception type to InvalidBSON but preserve traceback.
        _, exc_value, exc_tb = sys.exc_info()
        reraise(InvalidBSON, exc_value, exc_tb)
1127516
def params_to_mongo(query_params):
    
    if not query_params:
        return {}
    for key, value in query_params.items():
        if isinstance(value, list):
            query_params[key] = {'$in': value}
    return query_params
876920
def collect(self):
        """"""

        if pymongo is None:
            self.log.error('Unable to import pymongo')
            return

        hosts = self.config.get('hosts')

        # Convert a string config value to be an array
        if isinstance(hosts, basestring):
            hosts = [hosts]

        # we need this for backwards compatibility
        if 'host' in self.config:
            hosts = [self.config['host']]

        # convert network_timeout to integer
        if self.config['network_timeout']:
            self.config['network_timeout'] = int(
                self.config['network_timeout'])

        # convert collection_sample_rate to float
        if self.config['collection_sample_rate']:
            self.config['collection_sample_rate'] = float(
                self.config['collection_sample_rate'])

        # use auth if given
        if 'user' in self.config:
            user = self.config['user']
        else:
            user = None

        if 'passwd' in self.config:
            passwd = self.config['passwd']
        else:
            passwd = None

        for host in hosts:
            matches = re.search('((.+)\@)?(.+)?', host)
            alias = matches.group(2)
            host = matches.group(3)

            if alias is None:
                if len(hosts) == 1:
                    # one host only, no need to have a prefix
                    base_prefix = []
                else:
                    base_prefix = [re.sub('[:\.]', '_', host)]
            else:
                base_prefix = [alias]

            try:
                # Ensure that the SSL option is a boolean.
                if type(self.config['ssl']) is str:
                    self.config['ssl'] = str_to_bool(self.config['ssl'])

                if ReadPreference is None:
                    conn = pymongo.MongoClient(
                        host,
                        socketTimeoutMS=self.config['network_timeout'],
                        ssl=self.config['ssl'],
                    )
                else:
                    conn = pymongo.MongoClient(
                        host,
                        socketTimeoutMS=self.config['network_timeout'],
                        ssl=self.config['ssl'],
                        read_preference=ReadPreference.SECONDARY,
                    )
            except Exception as e:
                self.log.error('Couldnt connect to mongodb: %s', e)
                continue

            # try auth
            if user:
                try:
                    conn.admin.authenticate(user, passwd)
                except Exception as e:
                    self.log.error(
                        'User auth given, but could not autheticate' +
                        ' with host: %s, err: %s' % (host, e))
                    return{}

            data = conn.db.command('serverStatus')
            self._publish_transformed(data, base_prefix)
            if str_to_bool(self.config['simple']):
                data = self._extract_simple_data(data)
            if str_to_bool(self.config['replica']):
                try:
                    replset_data = conn.admin.command('replSetGetStatus')
                    self._publish_replset(replset_data, base_prefix)
                except pymongo.errors.OperationFailure as e:
                    self.log.error('error getting replica set status', e)

            self._publish_dict_with_prefix(data, base_prefix)
            db_name_filter = re.compile(self.config['databases'])
            ignored_collections = re.compile(self.config['ignore_collections'])
            sample_threshold = self.MAX_CRC32 * self.config[
                'collection_sample_rate']
            for db_name in conn.database_names():
                if not db_name_filter.search(db_name):
                    continue
                db_stats = conn[db_name].command('dbStats')
                db_prefix = base_prefix + ['databases', db_name]
                self._publish_dict_with_prefix(db_stats, db_prefix)
                for collection_name in conn[db_name].collection_names():
                    if ignored_collections.search(collection_name):
                        continue
                    if (self.config['collection_sample_rate'] < 1 and (
                            zlib.crc32(collection_name) & 0xffffffff
                    ) > sample_threshold):
                        continue

                    collection_stats = conn[db_name].command('collstats',
                                                             collection_name)
                    if str_to_bool(self.config['translate_collections']):
                        collection_name = collection_name.replace('.', '_')
                    collection_prefix = db_prefix + [collection_name]
                    self._publish_dict_with_prefix(collection_stats,
                                                   collection_prefix)
665611
def _bson_to_dict(data, opts):
    """"""
    try:
        obj_size = _UNPACK_INT(data[:4])[0]
    except struct.error as exc:
        raise InvalidBSON(str(exc))
    if obj_size != len(data):
        raise InvalidBSON("invalid object size")
    if data[obj_size - 1:obj_size] != b"\x00":
        raise InvalidBSON("bad eoo")
    try:
        return _elements_to_dict(data, 4, obj_size - 1, opts)
    except InvalidBSON:
        raise
    except Exception:
        # Change exception type to InvalidBSON but preserve traceback.
        _, exc_value, exc_tb = sys.exc_info()
        reraise(InvalidBSON, exc_value, exc_tb)
1349235
def to_json(df, values):
        """"""
        records = []
        if df.empty:
            return {"data": []}

        sum_ = float(np.sum([df[c].iloc[0] for c in values]))
        for c in values:
            records.append({
                "label": values[c],
                "value": "%.2f"%np.around(df[c].iloc[0] / sum_, decimals=2)
                })
        return {
            "data" : records
        }
1028275
def decode(cls, key):
        

        # Determine bucket key version
        prefix, sep, param_str = key.partition(':')
        if sep != ':' or prefix not in cls._prefix_to_version:
            raise ValueError("%r is not a bucket key" % key)
        version = cls._prefix_to_version[prefix]

        # Take the parameters apart...
        parts = param_str.split('/')
        uuid = parts.pop(0)
        params = {}
        for part in parts:
            name, sep, value = part.partition('=')

            # Make sure it's well-formed
            if sep != '=':
                raise ValueError("Cannot interpret parameter expression %r" %
                                 part)

            params[name] = cls._decode(value)

        # Return a BucketKey
        return cls(uuid, params, version=version)
145204
def decode_search_index(self, index):
        
        result = {}
        result['name'] = bytes_to_str(index.name)
        if index.HasField('schema'):
            result['schema'] = bytes_to_str(index.schema)
        if index.HasField('n_val'):
            result['n_val'] = index.n_val
        return result
547301
def decode_dict(data):
    
    if six.PY3:
        return data
    rv = {}
    for key, value in iteritems(data):
        if isinstance(key, text_type):
            key = key.encode('utf-8')

        if isinstance(value, text_type):
            value = value.encode('utf-8')
        elif isinstance(value, list):
            value = _decode_list(value)
        # no need to recurse into dict, json library will do that
        rv[key] = value
    return rv
170248
function (object) {
		if (!object || object._bsontype)
			return object;

		if (_.isString(object))
			return object.replace(/\./g, "U+FF0E").replace(/\$/g, "U+FF04");

		var newObject = _.isArray(object) ? [ ] : { };

		Object.keys(object).forEach(function (key) {
			var sKey = key;
			
			if (/[.|$]/.test(key)) {
				sKey = mongojs.util.sanitise(key);
				newObject[sKey] = object[key];
			} else {
				newObject[key] = object[key];
			}
			
			// Recurse
			if (isObject(newObject[sKey]))
				newObject[sKey] = this.sanitise(newObject[sKey]);
		}.bind(this));
		
		return newObject;
	}
597340
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'collection_id' in _dict:
            args['collection_id'] = _dict.get('collection_id')
        if 'name' in _dict:
            args['name'] = _dict.get('name')
        if 'description' in _dict:
            args['description'] = _dict.get('description')
        if 'created' in _dict:
            args['created'] = string_to_datetime(_dict.get('created'))
        if 'updated' in _dict:
            args['updated'] = string_to_datetime(_dict.get('updated'))
        if 'status' in _dict:
            args['status'] = _dict.get('status')
        if 'configuration_id' in _dict:
            args['configuration_id'] = _dict.get('configuration_id')
        if 'language' in _dict:
            args['language'] = _dict.get('language')
        if 'document_counts' in _dict:
            args['document_counts'] = DocumentCounts._from_dict(
                _dict.get('document_counts'))
        if 'disk_usage' in _dict:
            args['disk_usage'] = CollectionDiskUsage._from_dict(
                _dict.get('disk_usage'))
        if 'training_status' in _dict:
            args['training_status'] = TrainingStatus._from_dict(
                _dict.get('training_status'))
        if 'source_crawl' in _dict:
            args['source_crawl'] = SourceStatus._from_dict(
                _dict.get('source_crawl'))
        return cls(**args)