Go through the sequence of zone files discovered in a block range, and reindex the names' subdomains.
963378
def index_blockchain(self, block_start, block_end):
        
        log.debug("Processing subdomain updates for zonefiles in blocks {}-{}".format(block_start, block_end))
        
        res = self.find_zonefile_subdomains(block_start, block_end)
        zonefile_subdomain_info = res['zonefile_info']

        self.process_subdomains(zonefile_subdomain_info)
621397
def reindex(self, copy=True):
        """"""
        ret = deepcopy(self) if copy else self

        ret.stats = ret.stats.reindex(index=ret._idx, level=0)
        if ret.idx_depth == 2:
            ret.stats = ret.stats.reindex(index=ret._sub_idx, level=1)
        if ret.rows is not None:
            ret.stats = ret.stats.reindex(index=ret.rows, level=ret.idx_depth)

        ret.stats = ret.stats.reindex(columns=ret._headers, level=0)
        ret.stats = ret.stats.reindex(columns=ret._subheaders, level=1)
        ret.stats = ret.stats.reindex(columns=ret._describe_cols, level=2)

        if copy:
            return ret
43845
public void zone_zoneName_reset_POST(String zoneName, OvhResetRecord[] DnsRecords, Boolean minimized) throws IOException {
		String qPath = "/domain/zone/{zoneName}/reset";
		StringBuilder sb = path(qPath, zoneName);
		HashMap<String, Object>o = new HashMap<String, Object>();
		addBody(o, "DnsRecords", DnsRecords);
		addBody(o, "minimized", minimized);
		exec(qPath, "POST", sb.toString(), o);
	}
635500
def rescan(self):
        """"""
        self._pathfiles = {}
        for path in self.basepaths:
            self.scan_path(path)
193717
def parse_subdomain_missing_zonefiles_record(cls, rec):
        
        txt_entry = rec['txt']
        if isinstance(txt_entry, list):
            raise ParseError("TXT entry too long for a missing zone file list")

        try:
            return [int(i) for i in txt_entry.split(',')] if txt_entry is not None and len(txt_entry) > 0 else []
        except ValueError:
            raise ParseError('Invalid integers')
773276
def replace_blocks(self, blocks):
        """"""
        start = 0
        bulk_insert = self.bulk_insert
        blocks_len = len(blocks)
        select = 'SELECT ?,?,?,?,?'
        query = 'REPLACE INTO gauged_data (namespace, offset, `key`, ' \
            'data, flags) '
        execute = self.cursor.execute
        while start < blocks_len:
            rows = blocks[start:start+bulk_insert]
            params = [param for params in rows for param in params]
            insert = (select + ' UNION ') * (len(rows) - 1) + select
            execute(query + insert, params)
            start += bulk_insert
559665
def combine(self, blocks, copy=True):
        
        if len(blocks) == 0:
            return self.make_empty()

        # FIXME: optimization potential
        indexer = np.sort(np.concatenate([b.mgr_locs.as_array
                                          for b in blocks]))
        inv_indexer = lib.get_reverse_indexer(indexer, self.shape[0])

        new_blocks = []
        for b in blocks:
            b = b.copy(deep=copy)
            b.mgr_locs = algos.take_1d(inv_indexer, b.mgr_locs.as_array,
                                       axis=0, allow_fill=False)
            new_blocks.append(b)

        axes = list(self.axes)
        axes[0] = self.items.take(indexer)

        return self.__class__(new_blocks, axes, do_integrity_check=False)
320140
def subdomains_init(blockstack_opts, working_dir, atlas_state):
    
    if not is_subdomains_enabled(blockstack_opts):
        return None

    subdomain_state = SubdomainIndex(blockstack_opts['subdomaindb_path'], blockstack_opts=blockstack_opts)
    atlas_node_add_callback(atlas_state, 'store_zonefile', subdomain_state.enqueue_zonefile)

    return subdomain_state
321009
private void indexDomains() throws IOException {
		domainsFileLock.writeLock().lock();
		logger.trace("LOCK writelock");
		try {
			if( allDomains == null) {
				ensureDomainsFileInstalled();
			}

			// Leave enough space for all PDBs as of 2015
			domainMap = new HashMap<String, List<EcodDomain>>((int) (150000/.85),.85f);

			// Index with domainMap
			for(EcodDomain d : allDomains) {
				// Get the PDB ID, either directly or from the domain ID
				String pdbId = d.getPdbId();
				if( pdbId == null ) {
					String ecodId = d.getDomainId();
					if( ecodId != null && !ecodId.isEmpty() ) {
						Matcher match = ECOD_RE.matcher(ecodId);
						pdbId = match.group(1);
					}
				}

				// Add current domain to the map
				List<EcodDomain> currDomains;
				if( domainMap.containsKey(pdbId) ) {
					currDomains = domainMap.get(pdbId);
				} else {
					currDomains = new LinkedList<EcodDomain>();
					domainMap.put(pdbId,currDomains);
				}
				currDomains.add(d);
			}
		} finally {
			logger.trace("UNLOCK writelock");
			domainsFileLock.writeLock().unlock();
		}

	}
1293449
def read_ip_ranges(filename, local_file = True, ip_only = False, conditions = []):
    
    targets = []
    data = load_data(filename, local_file = local_file)
    if 'source' in data:
        # Filtered IP ranges
        conditions = data['conditions']
        local_file = data['local_file'] if 'local_file' in data else False
        data = load_data(data['source'], local_file = local_file, key_name = 'prefixes')
    else:
        # Plain IP ranges
        data = data['prefixes']
    for d in data:
        condition_passed = True
        for condition in conditions:
            if type(condition) != list or len(condition) < 3:
                continue
            condition_passed = pass_condition(d[condition[0]], condition[1], condition[2])
            if not condition_passed:
                break
        if condition_passed:
            targets.append(d)
    if ip_only:
        ips = []
        for t in targets:
            ips.append(t['ip_prefix'])
        return ips
    else:
        return targets
828329
def reformat(found_sequences):
    ''''''
    for (pdb_id, chain, file_name), sequence in sorted(found_sequences.iteritems()):
        header = sequence[0]
        assert(header[0] == '>')
        tokens = header.split('|')
        tokens[0] = tokens[0][:5]
        assert(len(tokens[0]) == 5)
        sequence[0] = "|".join(tokens)
1329467
void checkDifference() {
    resetDiffsAndStats();
    InjectionHandler.processEvent(InjectionEvent.DIRECTORY_SCANNER_BEFORE_FILE_SCAN);
    Map<Integer, DiskScanInfo[]> diskReportPerNamespace = getDiskReportPerNamespace();
    InjectionHandler.processEvent(InjectionEvent.DIRECTORY_SCANNER_AFTER_FILE_SCAN);
    try {
      for (Entry<Integer, DiskScanInfo[]> entry : diskReportPerNamespace.entrySet()) {

        Integer namespaceId = entry.getKey();
        DiskScanInfo[] namespaceReport = entry.getValue();

        Stats statsRecord = new Stats(namespaceId);
        statsPerNamespace.put(namespaceId, statsRecord);
        LinkedList<ScanDifference> diffRecords = new LinkedList<ScanDifference>();
        diffsPerNamespace.put(namespaceId, diffRecords);

        statsRecord.totalBlocks = namespaceReport.length;
        Block[] memReport = dataset.getBlockReport(namespaceId);
        Arrays.sort(memReport); // Sort based on blockId

        int d = 0; // index for blockpoolReport
        int m = 0; // index for memReprot
        while (m < memReport.length && d < namespaceReport.length) {
          Block memBlock = memReport[Math.min(m, memReport.length - 1)];
          DiskScanInfo scanInfo = namespaceReport[Math.min(
              d, namespaceReport.length - 1)];
          if (scanInfo.getBlockId() < memBlock.getBlockId()) {
            // Block is missing in memory
            // If this block was removed during scan, then do not add it to diff
            if (delta.get(namespaceId, scanInfo.getBlockId()) != BlockOperation.REMOVE) {
              // Otherwise this is the case for reconciliation
              statsRecord.missingMemoryBlocks++;
              statsRecord.missingMetaFile +=
                  scanInfo.getMetaFile() == null ? 1 : 0;
              statsRecord.missingBlockFile +=
                  scanInfo.getBlockFile() == null ? 1 : 0;
              diffRecords.add(ScanDifference.createDiffForMemoryBlockMissing(scanInfo));
            }
            d++;
            continue;
          }
          if (scanInfo.getBlockId() > memBlock.getBlockId()) {
            // Block is missing on the disk
            // If this block was added during scan, then do not add it to diff
            if (delta.get(namespaceId, memBlock) != BlockOperation.ADD) {
              statsRecord.missingBlockFile++;
              statsRecord.missingMetaFile++;
              diffRecords.add(ScanDifference.createDiffForDiskFilesMissing(memBlock.getBlockId()));
            }
            m++;
            continue;
          }
          // Block file and/or metadata file exists on the disk
          // Block exists in memory
          if (scanInfo.getBlockFile() == null) {
            // Block metadata file exits and block file is missing
            // this is not the case for reconciliation if the block was added or removed
            // during scanning process
            BlockOperation deltaOp = delta.get(namespaceId, memBlock);
            if (deltaOp != BlockOperation.ADD && deltaOp != BlockOperation.REMOVE) {
              statsRecord.missingBlockFile += scanInfo.getBlockFile() == null ? 1 : 0;
              diffRecords.add(ScanDifference.createDiffOutOfSync(scanInfo));
            }
          } else if (scanInfo.getGenStamp() != memBlock.getGenerationStamp()
              || scanInfo.getLength() != memBlock.getNumBytes()) {
            // Block metadata file is missing or has wrong generation stamp,
            // or block file length is different than expected. It could happen
            // in both add, remove and update operations
            if (delta.get(namespaceId, memBlock) == null) {
              statsRecord.mismatchBlocks++;
              statsRecord.missingMetaFile += scanInfo.getMetaFile() == null ? 1 : 0;
              diffRecords.add(ScanDifference.createDiffOutOfSync(scanInfo));
            }
          }
          d++;
          m++;
        }
        while (m < memReport.length) {
          if (delta.get(namespaceId, memReport[m].getBlockId()) != BlockOperation.ADD) {
            statsRecord.missingBlockFile++;
            statsRecord.missingMetaFile++;
            diffRecords
              .add(ScanDifference.createDiffForDiskFilesMissing(memReport[m].getBlockId()));
          }
          m++;
        }
        while (d < namespaceReport.length) {
          DiskScanInfo info = namespaceReport[d];
          if (delta.get(namespaceId, info.getBlockId()) != BlockOperation.REMOVE) {
            statsRecord.missingMemoryBlocks++;
            statsRecord.missingMetaFile += info.getMetaFile() == null ? 1 : 0;
            statsRecord.missingBlockFile += info.getBlockFile() == null ? 1 : 0;
            diffRecords.add(ScanDifference.createDiffForMemoryBlockMissing(info));
          }
          d++;
        }
      } //end for
    } catch (IOException e) {
      LOG.warn("Scanning failed beause of IOException", e);
    }
    InjectionHandler.processEvent(InjectionEvent.DIRECTORY_SCANNER_AFTER_DIFF);
    StringBuilder sb = new StringBuilder();
    for (Entry<Integer, Stats> entry : statsPerNamespace.entrySet()) {
      sb.append("Namespace ID: " + entry.getKey() + " results: " + entry.getValue().toString());
    }
    LOG.info(sb.toString());
  }
100737
def pack_subdomain(self):
        
        output = []

        # name (only fully-qualified if independent of the domain name)
        if self.independent:
            output.append(self.fqn)
        else:
            _, subdomain_name, _ = is_address_subdomain(self.fqn)
            output.append(subdomain_name)

        # address
        output.append(txt_encode_key_value(SUBDOMAIN_PUBKEY, self.address))

        # sequence number 
        output.append(txt_encode_key_value(SUBDOMAIN_N, "{}".format(self.n)))
        
        # subdomain zone file data, broken into 255-character base64 strings.
        # let's pack into 250 byte strings -- the entry "zf99=" eliminates 5 useful bytes,
        # and the max is 255.
        encoded_zf = base64.b64encode(self.zonefile_str)
        n_pieces = (len(encoded_zf) / 250) + 1
        if len(encoded_zf) % 250 == 0:
            n_pieces -= 1
        
        # number of pieces
        output.append(txt_encode_key_value(SUBDOMAIN_ZF_PARTS, "{}".format(n_pieces)))

        for i in range(n_pieces):
            start = i * 250
            piece_len = min(250, len(encoded_zf[start:]))
            assert piece_len != 0
            piece = encoded_zf[start:(start+piece_len)]

            # next piece
            output.append(txt_encode_key_value(SUBDOMAIN_ZF_PIECE % i, piece))

        # signature (optional)
        if self.sig is not None:
            output.append(txt_encode_key_value(SUBDOMAIN_SIG, self.sig))

        return output
636844
def get_subdomain_resolver(name, db_path=None, zonefiles_dir=None):
    
    opts = get_blockstack_opts()
    if not is_subdomains_enabled(opts):
        log.warn("Subdomain support is disabled")
        return None

    if db_path is None:
        db_path = opts['subdomaindb_path']

    if zonefiles_dir is None:
        zonefiles_dir = opts['zonefiles']

    db = SubdomainDB(db_path, zonefiles_dir)
    resolver_url = db.get_domain_resolver(name)

    return resolver_url
1054857
private void parseDomains() throws IOException {
		domainsFileLock.writeLock().lock();
		logger.trace("LOCK writelock");
		try {
			EcodParser parser = new EcodParser(getDomainFile());
			allDomains = parser.getDomains();
			parsedVersion = parser.getVersion();
		} finally {
			logger.trace("UNLOCK writelock");
			domainsFileLock.writeLock().unlock();
		}
	}
477309
def atlasdb_sync_zonefiles( db, start_block, zonefile_dir, atlas_state, validate=True, end_block=None, path=None, con=None ):
    
    ret = None
    with AtlasDBOpen(con=con, path=path) as dbcon:
        ret = atlasdb_queue_zonefiles( dbcon, db, start_block, zonefile_dir, validate=validate, end_block=end_block )
        atlasdb_cache_zonefile_info( con=dbcon )

    if atlas_state:
        # it could have been the case that a zone file we already have was re-announced.
        # if so, then inform any storage listeners in the crawler thread that this has happened
        # (such as the subdomain system).
        crawler_thread = atlas_state['zonefile_crawler']
        for zfinfo in filter(lambda zfi: zfi['present'], ret):
            log.debug('Store re-discovered zonefile {} at {}'.format(zfinfo['zonefile_hash'], zfinfo['block_height']))
            crawler_thread.store_zonefile_cb(zfinfo['zonefile_hash'], zfinfo['block_height'])

    return ret
457748
def remove_zone(self, zone_id):
        """"""
        updated_zones = []
        for zone in self.my_osid_object_form._my_map['zones']:
            if zone['id'] != zone_id:
                updated_zones.append(zone)
        self.my_osid_object_form._my_map['zones'] = updated_zones