Top level runner to load State and County  GeoJSON files into Mongo DB
543075
def run(self):
        
        logging.info("Starting GeoJSON MongoDB loading process.")
        mongo = dict(uri=self.mongo, db=self.db, collection=self.collection)
        self.load(self.source, **mongo)
        logging.info("Finished loading {0} into MongoDB".format(self.source))
53083
def loads(s):
    """"""
    _decoder = json.JSONDecoder()

    while s:
        s = s.strip()
        obj, pos = _decoder.raw_decode(s)
        if not pos:
            raise ValueError('no JSON object found at %i' % pos)
        yield obj
        s = s[pos:]
10696
def import_json(path, model, batch_len=100, db_alias='default', start_batch=0, end_batch=None, ignore_errors=False, verbosity=2):
    """"""

    # TODO: use a generator to save memory for large json files/databases
    if verbosity > 0:
        print('Reading json records (dictionaries) from {0}.'.format(repr(path)))
    item_list = json.load(open(path, 'r'))
    if verbosity > 0:
        print('Finished reading {0} items from {1}.'.format(len(item_list), repr(path)))
    import_items(item_list, model=model, batch_len=batch_len, db_alias=db_alias, start_batch=start_batch, end_batch=end_batch, ignore_errors=ignore_errors, verbosity=verbosity)
201676
function(mongo_host, db, collection) {

    var arrayMongo = function() {
    };

    arrayMongo.prototype = Array.prototype;

    extend(arrayMongo.prototype, {
        /**
         * Store the parameters
         */
        host: function() {
            return mongo_host;
        },
        db: function() {
            return db;
        },
        mongoCollection: function() {
            return collection;
        },
        load: function(finished) {
            console.log('HERE');
            var self = this;
            var cname = this.mongoCollection();
            mongoCache.getConnection(this.host(), this.db(), (function(err, db) {
                if(err) return;
                var collection = db.collection(cname);
                collection.find().toArray(function(err, docs) {
                    if(err || docs.length < 1) {
                        finished();
                        return;
                    }

                    // Copy all the documents in the array
                    docs.forEach(function(doc) {
                        self.push(doc);
                    });
                    finished();
                }.bind(this));
            }).bind(this));
        },
        save: function() {
            var self = this;
            var cname = this.mongoCollection();
            mongoCache.getConnection(this.host(), this.db(), (function(err, db) {
                if(err) return;
                var collection = db.collection(cname);
                self.forEach(function(element) {
                    if(element['_id']) {
                        collection.update({_id: element['_id']}, element, function(){
                        });
                    }
                    else {
                        collection.insert(element, function(){
                        });
                    }
                });
            }).bind(this));
        }
    });

    // Create instance and return it
    return new arrayMongo();
}
442772
protected void loadCitiesFileToTree(String gnObjectsFilename, Map<String, String> countryCodes){
		EasyBufferedReader reader = new EasyBufferedReader(gnObjectsFilename);
		String line = null;
		int count = 0;
		long t0 = System.currentTimeMillis();
		logger.info("loading of objects started");
		while ((line = reader.readLine()) != null){
			if (line.trim().length() < 1) continue;
			count++;
			GeoObject city = new GeoObject(line);
			
			double[] coords = new double[2];
			coords[0] = city.getLat();
			coords[1] = city.getLon();
			try {
				tree.insert(coords, new LightweightGeoObject(city));
			} catch (KeySizeException e) {
				logger.error(e.getMessage());
			} catch (KeyDuplicateException e) {
				logger.error(e.getMessage());
			}
		}
		logger.info(count + " objects loaded in " + (System.currentTimeMillis()-t0)/1000.0 + "secs");

		reader.close();
		logger.info("file " + gnObjectsFilename + " closed");
	}
597340
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'collection_id' in _dict:
            args['collection_id'] = _dict.get('collection_id')
        if 'name' in _dict:
            args['name'] = _dict.get('name')
        if 'description' in _dict:
            args['description'] = _dict.get('description')
        if 'created' in _dict:
            args['created'] = string_to_datetime(_dict.get('created'))
        if 'updated' in _dict:
            args['updated'] = string_to_datetime(_dict.get('updated'))
        if 'status' in _dict:
            args['status'] = _dict.get('status')
        if 'configuration_id' in _dict:
            args['configuration_id'] = _dict.get('configuration_id')
        if 'language' in _dict:
            args['language'] = _dict.get('language')
        if 'document_counts' in _dict:
            args['document_counts'] = DocumentCounts._from_dict(
                _dict.get('document_counts'))
        if 'disk_usage' in _dict:
            args['disk_usage'] = CollectionDiskUsage._from_dict(
                _dict.get('disk_usage'))
        if 'training_status' in _dict:
            args['training_status'] = TrainingStatus._from_dict(
                _dict.get('training_status'))
        if 'source_crawl' in _dict:
            args['source_crawl'] = SourceStatus._from_dict(
                _dict.get('source_crawl'))
        return cls(**args)
576586
public void loadFromJSONPlan(JSONObject jobj, Database db)  throws JSONException {
        if (jobj.has(Members.PLAN_NODES_LISTS)) {
            JSONArray jplanNodesArray = jobj.getJSONArray(Members.PLAN_NODES_LISTS);
            for (int i = 0; i < jplanNodesArray.length(); ++i) {
                JSONObject jplanNodesObj = jplanNodesArray.getJSONObject(i);
                JSONArray jplanNodes = jplanNodesObj.getJSONArray(Members.PLAN_NODES);
                int stmtId = jplanNodesObj.getInt(Members.STATEMENT_ID);
                loadPlanNodesFromJSONArrays(stmtId, jplanNodes, db);
            }
        }
        else {
            // There is only one statement in the plan. Its id is set to 0 by default
            int stmtId = 0;
            JSONArray jplanNodes = jobj.getJSONArray(Members.PLAN_NODES);
            loadPlanNodesFromJSONArrays(stmtId, jplanNodes, db);
        }

        // Connect the parent and child statements
        for (List<AbstractPlanNode> nextPlanNodes : m_planNodesListMap.values()) {
            for (AbstractPlanNode node : nextPlanNodes) {
                connectNodesIfNecessary(node);
            }
        }
    }
135335
def _read_geojson_features(data, features=None, prefix=""):
        """"""
        if features is None:
            features = collections.OrderedDict()
        for i, feature in enumerate(data['features']):
            key = feature.get('id', prefix + str(i))
            feature_type = feature['geometry']['type']
            if feature_type == 'FeatureCollection':
                _read_geojson_features(feature, features, prefix + '.' + key)
            elif feature_type == 'Point':
                value = Circle._convert_point(feature)
            elif feature_type in ['Polygon', 'MultiPolygon']:
                value = Region(feature)
            else:
                # TODO Support all http://geojson.org/geojson-spec.html#geometry-objects
                value = None
            features[key] = value
        return features
703676
def bulk_csv_import_mongo(csvfile, database_name, collection_name,
                          delete_collection_before_import=False):
    """method can be insert or update"""

    l = []
    response_dict = {}
    try:
        mongodb_client_url = getattr(settings, 'MONGODB_CLIENT',
                                 'mongodb://localhost:27017/')
        mc = MongoClient(mongodb_client_url,document_class=OrderedDict)
        db = mconnection[database_name]
        collection = db[collection_name]

        if delete_collection_before_import:
            myobjectid = collection.remove({})

        # open the csv file.
        csvhandle = csv.reader(open(csvfile._get_path(), 'rb'), delimiter=',')

        rowindex = 0
        errors = 0
        error_list = []
        success = 0
        for row in csvhandle:

            if rowindex == 0:
                column_headers = row
                cleaned_headers = []
                for c in column_headers:
                    c = c.replace(".", "")
                    c = c.replace("$", "-")
                    c = c.replace(" ", "_")
                    cleaned_headers.append(c)
            else:

                record = OrderedDict(zip(cleaned_headers, row))
                # if there is no values, skip the key value pair
                kwargs = OrderedDict()

                # Only populate fields that are not blank.
                for k, v in record.items():
                    if v:
                        if v.isdigit():
                            kwargs[k] = int(v)
                        else:
                            kwargs[k] = v
                try:

                    myobjectid = collection.insert(kwargs)
                    success += 1
                except:
                    error_message = "Error on row " + \
                        rowindex + ". " + str(sys.exc_info())
                    error_list.append(str(sys.exc_info()))

            rowindex += 1

        if error_list:
            response_dict = {}
            response_dict['num_rows_imported'] = rowindex
            response_dict['num_rows_errors'] = len(error_list)
            response_dict['errors'] = error_list
            response_dict['code'] = 400
            response_dict['message'] = "Completed with errors"
        else:

            response_dict = {}
            response_dict['num_rows_imported'] = success
            response_dict['code'] = 200
            response_dict['message'] = "Completed."
        return response_dict

    except:
        # print "Error reading from Mongo"
        # print str(sys.exc_info())
        response_dict['num_results'] = 0
        response_dict['code'] = 400
        response_dict['type'] = "Error"
        response_dict['results'] = []
        response_dict['message'] = str(sys.exc_info())
    return response_dict
776627
def source_files(mongodb_path):
    """"""
    for root, dirs, files in os.walk(mongodb_path):
        for filename in files:
            # skip files in dbtests folder
            if 'dbtests' in root:
                continue
            if filename.endswith(('.cpp', '.c', '.h')):
                yield os.path.join(root, filename)
1358835
def rebuild_collection_tree(old_collection_ident, new_document_id_map, plpy):
    
    get_tree = plpy.prepare('''
    WITH RECURSIVE t(node, parent, document, title, childorder, latest, path)
        AS (SELECT tr.nodeid, tr.parent_id, tr.documentid, tr.title,
                   tr.childorder, tr.latest, ARRAY[tr.nodeid]
            FROM trees tr
            WHERE tr.documentid = $1 AND tr.is_collated = 'False'
    UNION ALL
        SELECT c.nodeid, c.parent_id, c.documentid, c.title,
               c.childorder, c.latest, path || ARRAY[c.nodeid]
        FROM trees c JOIN t ON (c.parent_id = t.node)
        WHERE not c.nodeid = ANY(t.path)
    )
    SELECT * FROM t
    ''', ('integer',))

    def get_old_tree():
        return plpy.execute(get_tree, (old_collection_ident,))

    tree = {}  # { old_nodeid: {'data': ...}, ...}
    children = {}  # { nodeid: [child_nodeid, ...], child_nodeid: [...]}
    for i in get_old_tree():
        tree[i['node']] = {'data': i, 'new_nodeid': None}
        children.setdefault(i['parent'], [])
        children[i['parent']].append(i['node'])

    insert_tree = plpy.prepare('''
    INSERT INTO trees (nodeid, parent_id, documentid,
        title, childorder, latest)
    VALUES (DEFAULT, $1, $2, $3, $4, $5)
    RETURNING nodeid
    ''', ('integer', 'integer', 'text', 'integer', 'boolean'))

    def execute(fields):
        results = plpy.execute(insert_tree, fields, 1)
        return results[0]['nodeid']

    root_node = children[None][0]

    def build_tree(node, parent):
        data = tree[node]['data']
        new_node = execute([parent, new_document_id_map.get(data['document'],
                            data['document']), data['title'],
                            data['childorder'], data['latest']])
        for i in children.get(node, []):
            build_tree(i, new_node)
    build_tree(root_node, None)
124819
private <N, S extends State, SV> void migrateStateValues(
		StateDescriptor<S, SV> stateDesc,
		Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> stateMetaInfo) throws Exception {

		if (stateDesc.getType() == StateDescriptor.Type.MAP) {
			throw new StateMigrationException("The new serializer for a MapState requires state migration in order for the job to proceed." +
				" However, migration for MapState currently isn't supported.");
		}

		LOG.info(
			"Performing state migration for state {} because the state serializer's schema, i.e. serialization format, has changed.",
			stateDesc);

		// we need to get an actual state instance because migration is different
		// for different state types. For example, ListState needs to deal with
		// individual elements
		StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
		if (stateFactory == null) {
			String message = String.format("State %s is not supported by %s",
				stateDesc.getClass(), this.getClass());
			throw new FlinkRuntimeException(message);
		}
		State state = stateFactory.createState(
			stateDesc,
			stateMetaInfo,
			RocksDBKeyedStateBackend.this);
		if (!(state instanceof AbstractRocksDBState)) {
			throw new FlinkRuntimeException(
				"State should be an AbstractRocksDBState but is " + state);
		}

		@SuppressWarnings("unchecked")
		AbstractRocksDBState<?, ?, SV> rocksDBState = (AbstractRocksDBState<?, ?, SV>) state;

		Snapshot rocksDBSnapshot = db.getSnapshot();
		try (
			RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, stateMetaInfo.f0);
			RocksDBWriteBatchWrapper batchWriter = new RocksDBWriteBatchWrapper(db, getWriteOptions())
		) {
			iterator.seekToFirst();

			DataInputDeserializer serializedValueInput = new DataInputDeserializer();
			DataOutputSerializer migratedSerializedValueOutput = new DataOutputSerializer(512);
			while (iterator.isValid()) {
				serializedValueInput.setBuffer(iterator.value());

				rocksDBState.migrateSerializedValue(
					serializedValueInput,
					migratedSerializedValueOutput,
					stateMetaInfo.f1.getPreviousStateSerializer(),
					stateMetaInfo.f1.getStateSerializer());

				batchWriter.put(stateMetaInfo.f0, iterator.key(), migratedSerializedValueOutput.getCopyOfBuffer());

				migratedSerializedValueOutput.clear();
				iterator.next();
			}
		} finally {
			db.releaseSnapshot(rocksDBSnapshot);
			rocksDBSnapshot.close();
		}
	}
230563
def load(self):
        """"""
        basepath = os.path.dirname(os.path.abspath(__file__))
        filename = os.sep.join([basepath, c.FOLDER_JSON, c.FILE_GAME_VERSIONS])
        Handler.ALL_VERS_DATA = {} # reset known data; do not retain defunct information
        with open(filename, "r") as f:
            data = json.loads( f.read() )
        self.update(data)
        self._updated = False
621611
function writeJson( row ){
    let targetDir = path.join(dataDir, wofIdToPath(row.id).join(path.sep));
    try {
      fs.ensureDirSync(targetDir);
      fs.writeFileSync( path.join(targetDir, `${row.id}.geojson`), row.body, 'utf8' );
    } catch( error ){
      if( error ){ console.error(`error making directory ${targetDir}`); }
    }
  }
667165
function batchLoaderFunc(graphqlType, serializeRecordKey, getRecords) {
  return keys => getRecords(keys)
    .then(resultArray => getResultsByKey(
      keys, extractAllItems(resultArray), serializeRecordKey, graphqlType,
      { onError: logger }
      )
    );
}
1101955
function loadFormatters(){
    var arrFiles = glob.sync('./formatters/*.js', {
        'cwd': __dirname,
        'dot': false,
        'nodir': true,
        'strict': false,
        'silent': true
    });
    var mapFormatters = {};
    arrFiles.forEach(function(file){
        var fileInfo = path.parse(file);
        var formatterPath = path.resolve(__dirname, file);
        mapFormatters[fileInfo.name] = require(formatterPath);
    });
    return mapFormatters;
}
564649
function _resp2json(doc){
	console.log("_resp2json: " + doc);

	var kids = []; // ret

	// Extract the intersting graphs.
	var topo_graph_field = 'topology_graph_json';
	var trans_graph_field = 'regulates_transitivity_graph_json';
	var topo_graph = new model.graph();
	topo_graph.load_base_json(JSON.parse(doc[topo_graph_field]));
	var trans_graph = new model.graph();
	trans_graph.load_base_json(JSON.parse(doc[trans_graph_field]));

	//console.log('topo: ' + doc[topo_graph_field]);

	// Collect child ids.
	var child_ids = [];
	us.each(topo_graph.get_child_nodes(doc['id']), function(kid){
	    child_ids.push(kid.id());
	});

	// Using the badger, get the spans for the IDs.
	var ac_badges = new CountBadges(filter_manager);
	var ids_to_badge_text = ac_badges.get_future_badges(child_ids);

	//
	us.each(topo_graph.get_child_nodes(doc['id']), function(kid){

	    // Extract some info.
	    var oid = doc['id'];
	    var sid = kid.id();
	    var preds = topo_graph.get_predicates(sid, oid);
	    var imgsrc = bbop.resourcify(sd.image_base, preds[0], 'gif');

	    var lbl_txt =  kid.label() || sid;
	    
	    // Push template.
	    kids.push({
		'id': sid,
		'text': lbl_txt + ids_to_badge_text[sid],
		'icon': imgsrc,
		'state': {
		    'opened': false,
		    'disabled': false,
		    'selected': false
		},
		'children': true, // unknowable w/o lookahead
		'li_attr': {},
		'a_attr': {}
	    });
	});

	return kids;
    }