Set up subdomain state
    Returns a SubdomainIndex object that has been successfully connected to Atlas
320140
def subdomains_init(blockstack_opts, working_dir, atlas_state):
    
    if not is_subdomains_enabled(blockstack_opts):
        return None

    subdomain_state = SubdomainIndex(blockstack_opts['subdomaindb_path'], blockstack_opts=blockstack_opts)
    atlas_node_add_callback(atlas_state, 'store_zonefile', subdomain_state.enqueue_zonefile)

    return subdomain_state
1055755
def atlas_node_init(my_hostname, my_portnum, atlasdb_path, zonefile_dir, working_dir):
    
    atlas_state = {}
    atlas_state['peer_crawler'] = AtlasPeerCrawler(my_hostname, my_portnum, atlasdb_path, working_dir)
    atlas_state['health_checker'] = AtlasHealthChecker(my_hostname, my_portnum, atlasdb_path)
    atlas_state['zonefile_crawler'] = AtlasZonefileCrawler(my_hostname, my_portnum, atlasdb_path, zonefile_dir)
    # atlas_state['zonefile_pusher'] = AtlasZonefilePusher(my_hostname, my_portnum, atlasdb_path, zonefile_dir)

    return atlas_state
1179957
def get_all_subdomains(offset=None, count=None, min_sequence=None, db_path=None, zonefiles_dir=None):
    
    opts = get_blockstack_opts()
    if not is_subdomains_enabled(opts):
        return []

    if db_path is None:
        db_path = opts['subdomaindb_path']

    if zonefiles_dir is None:
        zonefiles_dir = opts['zonefiles']

    db = SubdomainDB(db_path, zonefiles_dir)
    return db.get_all_subdomains(offset=offset, count=count, min_sequence=None)
321009
private void indexDomains() throws IOException {
		domainsFileLock.writeLock().lock();
		logger.trace("LOCK writelock");
		try {
			if( allDomains == null) {
				ensureDomainsFileInstalled();
			}

			// Leave enough space for all PDBs as of 2015
			domainMap = new HashMap<String, List<EcodDomain>>((int) (150000/.85),.85f);

			// Index with domainMap
			for(EcodDomain d : allDomains) {
				// Get the PDB ID, either directly or from the domain ID
				String pdbId = d.getPdbId();
				if( pdbId == null ) {
					String ecodId = d.getDomainId();
					if( ecodId != null && !ecodId.isEmpty() ) {
						Matcher match = ECOD_RE.matcher(ecodId);
						pdbId = match.group(1);
					}
				}

				// Add current domain to the map
				List<EcodDomain> currDomains;
				if( domainMap.containsKey(pdbId) ) {
					currDomains = domainMap.get(pdbId);
				} else {
					currDomains = new LinkedList<EcodDomain>();
					domainMap.put(pdbId,currDomains);
				}
				currDomains.add(d);
			}
		} finally {
			logger.trace("UNLOCK writelock");
			domainsFileLock.writeLock().unlock();
		}

	}
381317
def update_subdomain_entry(self, subdomain_obj, cur=None):
        
        # sanity checks
        assert isinstance(subdomain_obj, Subdomain)
       
        # NOTE: there is no need to call fsync() on the zone file fd here---we already have the data from the on-chain name's zone file fsync'ed,
        # so this information is already durable (albeit somewhere else) and can ostensibly be restored later.
        # We get such high subdomain traffic that we cannot call fsync() here each time; otherwise we could stall the node.
        zonefile_hash = get_zonefile_data_hash(subdomain_obj.zonefile_str)
        rc = store_atlas_zonefile_data(subdomain_obj.zonefile_str, self.zonefiles_dir, fsync=False)
        if not rc:
            raise Exception("Failed to store zone file {} from {}".format(zonefile_hash, subdomain_obj.get_fqn()))
        
        write_cmd = 'INSERT OR REPLACE INTO {} VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)'.format(self.subdomain_table)
        args = (subdomain_obj.get_fqn(), subdomain_obj.domain, subdomain_obj.n, subdomain_obj.address, zonefile_hash,
                subdomain_obj.sig, subdomain_obj.block_height, subdomain_obj.parent_zonefile_hash,
                subdomain_obj.parent_zonefile_index, subdomain_obj.zonefile_offset, subdomain_obj.txid, 
                ','.join(str(i) for i in subdomain_obj.domain_zonefiles_missing),
                1 if subdomain_obj.accepted else 0,
                subdomain_obj.resolver)

        cursor = None
        if cur is None:
            cursor = self.conn.cursor()
        else:
            cursor = cur

        db_query_execute(cursor, write_cmd, args)
        num_rows_written = cursor.rowcount
        
        if cur is None:
            # not part of a transaction
            self.conn.commit()

        if num_rows_written != 1:
            raise ValueError("No row written: fqn={} seq={}".format(subdomain_obj.get_fqn(), subdomain_obj.n))

        return True
932547
def is_subdomain(domain, reference):
    
    index_of_reference = domain.find(reference)
    if index_of_reference > 0 and domain[index_of_reference:] == reference:
        return True
    return False
926433
def fetch_namespace():
    
    resp = blockstack.lib.client.get_all_names(hostport=blockstack_indexer_url)
    subdomain_names = blockstack.lib.subdomains.get_all_subdomains()

    all_names = list(resp) + list(subdomain_names)

    with open(SEARCH_BLOCKCHAIN_DATA_FILE, 'w') as fout:
        fout.write(json.dumps(all_names))
599302
def get_name_history_page(name, page, hostport=None, proxy=None):
    
    assert hostport or proxy, 'Need hostport or proxy'
    if proxy is None:
        proxy = connect_hostport(hostport)

    name_required = ['op', 'opcode', 'txid', 'vtxindex']
    subdomain_required = ['txid']
    required = None
    if check_subdomain(name):
        log.debug('Subdomain: {}'.format(name))
        required = subdomain_required
    else:
        log.debug('Name: {}'.format(name))
        required = name_required

    hist_schema = {
        'type': 'object',
        'patternProperties': {
            '^[0-9]+$': {
                'type': 'array',
                'items': {
                    'type': 'object',
                    'properties': OP_HISTORY_SCHEMA['properties'],
                    'required': required
                },
            },
        },
    }

    hist_resp_schema = {
        'type': 'object',
        'properties': {
            'history': hist_schema,
        },
        'required': [ 'history' ],
    }

    resp_schema = json_response_schema(hist_resp_schema)
    resp = {}
    lastblock = None
    indexin = None

    try:
        _resp = proxy.get_name_history_page(name, page)
        resp = json_validate(resp_schema, _resp)
        if json_is_error(resp):
            return resp

        lastblock = _resp['lastblock']
        indexing = _resp['indexing']

    except ValidationError as e:
        if BLOCKSTACK_DEBUG:
            log.exception(e)

        resp = json_traceback(resp.get('error'))
        return resp

    except socket.timeout:
        log.error("Connection timed out")
        resp = {'error': 'Connection to remote host timed out.', 'http_status': 503}
        return resp

    except socket.error as se:
        log.error("Connection error {}".format(se.errno))
        resp = {'error': 'Connection to remote host failed.', 'http_status': 502}
        return resp

    except Exception as ee:
        if BLOCKSTACK_DEBUG:
            log.exception(ee)

        log.error("Caught exception while connecting to Blockstack node: {}".format(ee))
        resp = {'error': 'Failed to contact Blockstack node.  Try again with `--debug`.'}
        return resp

    return {'status': True, 'history': resp['history'], 'lastblock': lastblock, 'indexing': indexing}
965654
def get_host(self, host):
        '''  '''

        if len(self.index) == 0:
            # Need to load index from cache
            self.load_index_from_cache()

        if not host in self.index:
            # try updating the cache
            self.do_api_calls_update_cache()
            if not host in self.index:
                # host might not exist anymore
                return {}

        (region, instance_id) = self.index[host]

        instance = self.get_instance(region, instance_id)
        return self.get_host_info_dict_from_instance(instance)
557311
def _set_client(self):
        """"""
        if self._client is None:
            if mongo_proxy:
                self._client = mongo_proxy.MongoProxy(
                    pymongo.MongoClient(self.connection_string),
                    logger=LOG)
            else:
                LOG.warning("MongoDBProxy not imported. AutoReconnect "
                            "is not enabled.")
                self._client = mongo_proxy.MongoProxy(
                    pymongo.MongoClient(self.connection_string),
                    logger=LOG)
            LOG.debug("Created new connection to MongoDB: %s",
                      self.safe_connection_string)
953645
def get_DID_subdomain(did, db_path=None, zonefiles_dir=None, atlasdb_path=None, check_pending=False):
    
    opts = get_blockstack_opts()
    if not is_subdomains_enabled(opts):
        log.warn("Subdomain support is disabled")
        return None

    if db_path is None:
        db_path = opts['subdomaindb_path']

    if zonefiles_dir is None:
        zonefiles_dir = opts['zonefiles']
    
    if atlasdb_path is None:
        atlasdb_path = opts['atlasdb_path']

    db = SubdomainDB(db_path, zonefiles_dir)
    try:
        subrec = db.get_DID_subdomain(did)
    except Exception as e:
        if BLOCKSTACK_DEBUG:
            log.exception(e)

        log.warn("Failed to load subdomain for {}".format(did))
        return None

    if check_pending:
        # make sure that all of the zone files between this subdomain's
        # domain's creation and this subdomain's zone file index are present,
        # minus the ones that are allowed to be missing.
        subrec.pending = db.subdomain_check_pending(subrec, atlasdb_path)

    return subrec
445858
def get_host_map(root):
  ''' 
  '''
  hosts_map = {}
  for host in root.get_all_hosts():
    hosts_map[host.hostId] = {"hostname": NAGIOS_HOSTNAME_FORMAT % (host.hostname,),
                              "address": host.ipAddress}

  ''' Also define "virtual hosts" for the CM clusters- they will be the hosts
      to which CM services are mapped
  '''
  for cluster in root.get_all_clusters():
    hosts_map[cluster.name] = {"hostname": cluster.name,
                               "address": quote(cluster.name)}
  hosts_map[CM_DUMMY_HOST] = {"hostname": CM_DUMMY_HOST,
                              "address": CM_DUMMY_HOST}
  return hosts_map
253456
def __init(self):
        
        params = {
            "f" : "json",
        }
        json_dict = self._get(self._url, params,
                                 securityHandler=self._securityHandler,
                                 proxy_url=self._proxy_url,
                                 proxy_port=self._proxy_port)
        self._json_dict = json_dict
        self._json = json.dumps(self._json_dict)
        attributes = [attr for attr in dir(self)
                      if not attr.startswith('__') and \
                      not attr.startswith('_')]
        for k,v in json_dict.items():
            if k in attributes:
                if k == "versions" and json_dict[k]:
                    self._versions = []
                    for version in v:
                        self._versions.append(
                            Version(url=self._url + "/versions/%s" % version,
                                    securityHandler=self._securityHandler,
                                    proxy_url=self._proxy_url,
                                    proxy_port=self._proxy_port,
                                    initialize=False))
                elif k == "replicas" and json_dict[k]:
                    self._replicas = []
                    for version in v:
                        self._replicas.append(
                            Replica(url=self._url + "/replicas/%s" % version,
                                    securityHandler=self._securityHandler,
                                    proxy_url=self._proxy_url,
                                    proxy_port=self._proxy_port,
                                    initialize=False))
                else:
                    setattr(self, "_"+ k, v)
            else:
                print (k, " - attribute not implemented for GeoData Service")
97680
def is_subdomains_enabled(blockstack_opts):
    
    if not is_atlas_enabled(blockstack_opts):
        log.debug("Subdomains are disabled")
        return False

    if 'subdomaindb_path' not in blockstack_opts:
        log.debug("Subdomains are disabled: no 'subdomaindb_path' path set")
        return False

    return True
910882
def subscribeContext(self, objectID, domain, dist, varIDs=(
            tc.VAR_ROAD_ID, tc.VAR_LANEPOSITION), begin=0, end=2**31 - 1):
        
        Domain.subscribeContext(
            self, objectID, domain, dist, varIDs, begin, end)
204462
def setup(self):
        
        # Get the existing mapping & cache it. We'll compare it
        # during the ``update`` & if it doesn't match, we'll put the new
        # mapping.
        for language in self.languages:
            self.index_name = self._index_name_for_language(language)
            try:
                self.existing_mapping[language] = self.conn.indices.get_mapping(
                    index=self.index_name)
            except NotFoundError:
                pass
            except Exception:
                if not self.silently_fail:
                    raise

            unified_index = haystack.connections[self.connection_alias].get_unified_index()

            self.content_field_name, field_mapping = self.build_schema(
                unified_index.all_searchfields(), language)

            current_mapping = {
                'modelresult': {
                    'properties': field_mapping,
                    '_boost': {
                        'name': 'boost',
                        'null_value': 1.0
                    }
                }
            }

            if current_mapping != self.existing_mapping[language]:
                try:
                    # Make sure the index is there first.
                    self.conn.indices.create(
                        index=self.index_name,
                        body=self.DEFAULT_SETTINGS,
                        ignore=400)
                    self.conn.indices.put_mapping(
                        index=self.index_name,
                        doc_type='modelresult',
                        body=current_mapping
                    )
                    self.existing_mapping[language] = current_mapping
                except Exception:
                    if not self.silently_fail:
                        raise

        self.setup_complete = True
691081
def atlas_peer_getinfo( peer_hostport, timeout=None, peer_table=None ):
    

    if timeout is None:
        timeout = atlas_ping_timeout()

    host, port = url_to_host_port( peer_hostport )
    RPC = get_rpc_client_class()
    rpc = RPC( host, port, timeout=timeout )

    assert not atlas_peer_table_is_locked_by_me()

    log.debug("getinfo %s" % peer_hostport)
    res = None

    try:
        res = blockstack_getinfo( proxy=rpc )
        if 'error' in res:
            log.error("Failed to getinfo on %s: %s" % (peer_hostport, res['error']))

        if 'last_block_processed' not in res:
            log.error("Missing last_block_processed response from {}".format(peer_hostport))
            res = {'error': 'Remote peer {} did not rely last_block_processed'.format(peer_hostport)}

        if 'stale' in res and res['stale']:
            log.error("Remote host {} is well behind the chain tip".format(peer_hostport))
            res = {'error': 'Remote peer {} is well behind the chain tip'.format(peer_hostport)}

        if 'testnet' in res:
            if (res['testnet'] and not BLOCKSTACK_TEST and not BLOCKSTACK_PUBLIC_TESTNET) or (not res['testnet'] and (BLOCKSTACK_TEST or BLOCKSTACK_PUBLIC_TESTNET)):
                if BLOCKSTACK_TEST or BLOCKSTACK_PUBLIC_TESTNET:
                    log.error("Remote host {} is a mainnet host, and we're testnet".format(peer_hostport))
                    res = {'error': 'Remote peer {} is a mainnet host, and we\'re testnet'.format(peer_hostport)}
                else:
                    log.error("Remote host {} is a testnet host".format(peer_hostport))
                    res = {'error': 'Remote peer {} is a testnet host'.format(peer_hostport)}

        else:
            # assume mainnet 
            if BLOCKSTACK_TEST or BLOCKSTACK_PUBLIC_TESTNET:
                log.error("Remote host {} is a mainnet host, and we're testnet".format(peer_hostport))
                res = {'error': 'Remote peer {} is a mainnet host, and we\'re testnet'.format(peer_hostport)}

    except (socket.timeout, socket.gaierror, socket.herror, socket.error), se:
        atlas_log_socket_error( "getinfo(%s)" % peer_hostport, peer_hostport, se )

    except AssertionError, ae:
        log.exception(ae)
        log.error("Invalid server reply for getinfo from %s" % peer_hostport)

    except Exception, e:
        log.exception(e)
        log.error("Failed to get response from %s" % peer_hostport)
  
    if res is not None:
        err = False
        if json_is_error(res):
            log.error("Failed to contact {}: replied error '{}'".format(peer_hostport, res['error']))
            err = True
    
        if 'stale' in res and res['stale']:
            # peer is behind the chain tip
            log.warning("Peer {} reports that it is too far behind the chain tip.  Ignoring for now.".format(peer_hostport))
            err = True

        if err:
            res = None

    else:
        log.error("Failed to contact {}: no response".format(peer_hostport))

    # update health
    with AtlasPeerTableLocked(peer_table) as ptbl:
        if ptbl.has_key(peer_hostport):
            atlas_peer_update_health( peer_hostport, (res is not None), peer_table=ptbl )

    return res