Parse a missing-zonefiles vector given by the domain.
        Returns the list of zone file indexes on success
        Raises ParseError on unparseable records
193717
def parse_subdomain_missing_zonefiles_record(cls, rec):
        
        txt_entry = rec['txt']
        if isinstance(txt_entry, list):
            raise ParseError("TXT entry too long for a missing zone file list")

        try:
            return [int(i) for i in txt_entry.split(',')] if txt_entry is not None and len(txt_entry) > 0 else []
        except ValueError:
            raise ParseError('Invalid integers')
1213448
def _list_records_in_zone(self, zone, rdtype=None, name=None, content=None):
        
        records = []
        rrsets = zone.iterate_rdatasets() if zone else []
        for rname, rdataset in rrsets:
            rtype = dns.rdatatype.to_text(rdataset.rdtype)
            if ((not rdtype or rdtype == rtype)
                    and (not name or name == rname.to_text())):
                for rdata in rdataset:
                    rdata = rdata.to_text()
                    if not content or self._convert_content(rtype, content) == rdata:
                        raw_rdata = self._clean_TXT_record({'type': rtype,
                                                            'content': rdata})['content']
                        data = {
                            'type': rtype,
                            'name': rname.to_text(True),
                            'ttl': int(rdataset.ttl),
                            'content': raw_rdata,
                            'id': Provider._create_identifier(rtype, rname.to_text(), raw_rdata)
                        }
                        records.append(data)
        return records
1198436
def _extract_subdomain(self, rowdata):
        
        name = str(rowdata['fully_qualified_subdomain'])
        domain = str(rowdata['domain'])
        n = str(rowdata['sequence'])
        encoded_pubkey = str(rowdata['owner'])
        zonefile_hash = str(rowdata['zonefile_hash'])
        sig = rowdata['signature']
        block_height = int(rowdata['block_height'])
        parent_zonefile_hash = str(rowdata['parent_zonefile_hash'])
        parent_zonefile_index = int(rowdata['parent_zonefile_index'])
        zonefile_offset = int(rowdata['zonefile_offset'])
        txid = str(rowdata['txid'])
        missing = [int(i) for i in rowdata['missing'].split(',')] if rowdata['missing'] is not None and len(rowdata['missing']) > 0 else []
        accepted = int(rowdata['accepted'])
        resolver = str(rowdata['resolver']) if rowdata['resolver'] is not None else None

        if accepted == 0:
            accepted = False
        else:
            accepted = True

        if sig == '' or sig is None:
            sig = None
        else:
            sig = str(sig)

        name = str(name)
        is_subdomain, _, _ = is_address_subdomain(name)
        if not is_subdomain:
            raise Exception("Subdomain DB lookup returned bad subdomain result {}".format(name))

        zonefile_str = get_atlas_zonefile_data(zonefile_hash, self.zonefiles_dir)
        if zonefile_str is None:
            log.error("No zone file for {}".format(name))
            raise SubdomainNotFound('{}: missing zone file {}'.format(name, zonefile_hash))

        return Subdomain(str(name), str(domain), str(encoded_pubkey), int(n), str(zonefile_str), sig, block_height, parent_zonefile_hash, parent_zonefile_index, zonefile_offset, txid, domain_zonefiles_missing=missing, accepted=accepted, resolver=resolver)
1277789
def parse_file(self, filename=None, buffer=None, fileobj=None):
        """"""
        if filename:
            self.file = GzipFile(filename, 'rb')
        elif buffer:
            if hasattr(buffer, 'name'):
                self.filename = buffer.name
            self.file = buffer
        elif fileobj:
            if hasattr(fileobj, 'name'):
                self.filename = fileobj.name
            self.file = GzipFile(fileobj=fileobj)
        if self.file:
            try:
                type = TAG_Byte(buffer=self.file)
                if type.value == self.id:
                    name = TAG_String(buffer=self.file).value
                    self._parse_buffer(self.file)
                    self.name = name
                    self.file.close()
                else:
                    raise MalformedFileError(
                        "First record is not a Compound Tag")
            except StructError as e:
                raise MalformedFileError(
                    "Partial File Parse: file possibly truncated.")
        else:
            raise ValueError(
                "NBTFile.parse_file(): Need to specify either a "
                "filename or a file object"
            )
1164313
def hostedzone_from_element(zone):
    
    return HostedZone(
        name=maybe_bytes_to_unicode(zone.find("Name").text).encode("ascii").decode("idna"),
        identifier=maybe_bytes_to_unicode(zone.find("Id").text).replace(u"/hostedzone/", u""),
        rrset_count=int(zone.find("ResourceRecordSetCount").text),
        reference=maybe_bytes_to_unicode(zone.find("CallerReference").text),
    )
95907
def get_zone(self, zone_id, records=True):
        
        mask = None
        if records:
            mask = 'resourceRecords'
        return self.service.getObject(id=zone_id, mask=mask)
1338923
def parse_zone(zonefile=None, zone=None):
    '''
    
    '''
    if zonefile:
        try:
            with salt.utils.files.fopen(zonefile, 'r') as fp_:
                zone = salt.utils.stringutils.to_unicode(fp_.read())
        except Exception:
            pass

    if not zone:
        return 'Error: Zone data was not found'

    zonedict = {}
    mode = 'single'
    for line in zone.splitlines():
        comps = line.split(';')
        line = comps[0].strip()
        if not line:
            continue
        comps = line.split()
        if line.startswith('$'):
            zonedict[comps[0].replace('$', '')] = comps[1]
            continue
        if '(' in line and ')' not in line:
            mode = 'multi'
            multi = ''
        if mode == 'multi':
            multi += ' {0}'.format(line)
            if ')' in line:
                mode = 'single'
                line = multi.replace('(', '').replace(')', '')
            else:
                continue
        if 'ORIGIN' in zonedict:
            comps = line.replace('@', zonedict['ORIGIN']).split()
        else:
            comps = line.split()
        if 'SOA' in line:
            if comps[1] != 'IN':
                comps.pop(1)
            zonedict['ORIGIN'] = comps[0]
            zonedict['NETWORK'] = comps[1]
            zonedict['SOURCE'] = comps[3]
            zonedict['CONTACT'] = comps[4].replace('.', '@', 1)
            zonedict['SERIAL'] = comps[5]
            zonedict['REFRESH'] = _to_seconds(comps[6])
            zonedict['RETRY'] = _to_seconds(comps[7])
            zonedict['EXPIRE'] = _to_seconds(comps[8])
            zonedict['MINTTL'] = _to_seconds(comps[9])
            continue
        if comps[0] == 'IN':
            comps.insert(0, zonedict['ORIGIN'])
        if not comps[0].endswith('.') and 'NS' not in line:
            comps[0] = '{0}.{1}'.format(comps[0], zonedict['ORIGIN'])
        if comps[2] == 'NS':
            zonedict.setdefault('NS', []).append(comps[3])
        elif comps[2] == 'MX':
            if 'MX' not in zonedict:
                zonedict.setdefault('MX', []).append({'priority': comps[3],
                                                      'host': comps[4]})
        elif comps[3] in ('A', 'AAAA'):
            zonedict.setdefault(comps[3], {})[comps[0]] = {
                'TARGET': comps[4],
                'TTL': comps[1],
            }
        else:
            zonedict.setdefault(comps[2], {})[comps[0]] = comps[3]
    return zonedict
169505
def __fetch_route53_zone_records(self, zone_id):
        
        route53 = self.session.client('route53')

        done = False
        nextName = nextType = None
        records = {}

        try:
            while not done:
                if nextName and nextType:
                    response = route53.list_resource_record_sets(
                        HostedZoneId=zone_id,
                        StartRecordName=nextName,
                        StartRecordType=nextType
                    )
                else:
                    response = route53.list_resource_record_sets(HostedZoneId=zone_id)

                if response['IsTruncated']:
                    nextName = response['NextRecordName']
                    nextType = response['NextRecordType']
                else:
                    done = True

                if 'ResourceRecordSets' in response:
                    for record in response['ResourceRecordSets']:
                        # Cannot make this a list, due to a race-condition in the AWS api that might return the same
                        # record more than once, so we use a dict instead to ensure that if we get duplicate records
                        # we simply just overwrite the one already there with the same info.
                        record_id = self._get_resource_hash(zone_id, record)
                        if 'AliasTarget' in record:
                            value = record['AliasTarget']['DNSName']
                            records[record_id] = {
                                'id': record_id,
                                'name': record['Name'].rstrip('.'),
                                'type': 'ALIAS',
                                'ttl': 0,
                                'value': [value]
                            }
                        else:
                            value = [y['Value'] for y in record['ResourceRecords']]
                            records[record_id] = {
                                'id': record_id,
                                'name': record['Name'].rstrip('.'),
                                'type': record['Type'],
                                'ttl': record['TTL'],
                                'value': value
                            }

            return list(records.values())
        finally:
            del route53
1062264
def get_subdomain_entry_at_zonefile_index(self, fqn, zonefile_index, cur=None):
        
        get_cmd = "SELECT * FROM {} WHERE fully_qualified_subdomain=? AND parent_zonefile_index=?".format(self.subdomain_table)
        cursor = None
        if cur is None:
            cursor = self.conn.cursor()
        else:
            cursor = cur

        db_query_execute(cursor, get_cmd, (fqn,zonefile_index))

        try:
            rowdata = cursor.fetchone()
            assert rowdata
        except Exception as e:
            raise SubdomainNotFound(fqn)

        return self._extract_subdomain(rowdata)
145268
def get_zonefiles(hostport, zonefile_hashes, timeout=30, my_hostport=None, proxy=None):
    

    assert hostport or proxy, 'need either hostport or proxy'

    zonefiles_schema = {
        'type': 'object',
        'properties': {
            'zonefiles': {
                'type': 'object',
                'patternProperties': {
                    OP_ZONEFILE_HASH_PATTERN: {
                        'type': 'string',
                        'pattern': OP_BASE64_EMPTY_PATTERN
                    },
                },
            },
        },
        'required': [
            'zonefiles',
        ]
    }

    schema = json_response_schema( zonefiles_schema )

    if proxy is None:
        proxy = connect_hostport(hostport)

    zonefiles = None
    try:
        zf_payload = proxy.get_zonefiles(zonefile_hashes)
        zf_payload = json_validate(schema, zf_payload)
        if json_is_error(zf_payload):
            return zf_payload

        decoded_zonefiles = {}

        for zf_hash, zf_data_b64 in zf_payload['zonefiles'].items():
            zf_data = base64.b64decode( zf_data_b64 )
            assert verify_zonefile( zf_data, zf_hash ), "Zonefile data mismatch"

            # valid
            decoded_zonefiles[ zf_hash ] = zf_data

        # return this
        zf_payload['zonefiles'] = decoded_zonefiles
        zonefiles = zf_payload

    except AssertionError as ae:
        if BLOCKSTACK_DEBUG:
            log.exception(ae)

        zonefiles = {'error': 'Zonefile data mismatch'}
        return zonefiles

    except socket.timeout:
        log.error("Connection timed out")
        resp = {'error': 'Connection to remote host timed out.', 'http_status': 503}
        return resp

    except socket.error as se:
        log.error("Connection error {}".format(se.errno))
        resp = {'error': 'Connection to remote host failed.', 'http_status': 502}
        return resp

    except ValidationError as ve:
        if BLOCKSTACK_DEBUG:
            log.exception(ve)

        zonefiles = {'error': 'Server response did not match expected schema.  You are likely communicating with an out-of-date Blockstack node.', 'http_status': 502}
        return zonefiles

    except Exception as ee:
        if BLOCKSTACK_DEBUG:
            log.exception(ee)

        log.error("Caught exception while connecting to Blockstack node: {}".format(ee))
        resp = {'error': 'Failed to contact Blockstack node.  Try again with `--debug`.', 'http_status': 500}
        return resp

    return zonefiles
1022372
def atlas_get_zonefiles( my_hostport, peer_hostport, zonefile_hashes, timeout=None, peer_table=None ):
    

    if timeout is None:
        timeout = atlas_zonefiles_timeout()

    zf_payload = None
    zonefile_datas = {}

    host, port = url_to_host_port( peer_hostport )
    RPC = get_rpc_client_class()
    rpc = RPC( host, port, timeout=timeout, src=my_hostport )

    assert not atlas_peer_table_is_locked_by_me()

    # get in batches of 100 or less 
    zf_batches = []
    for i in xrange(0, len(zonefile_hashes), 100):
        zf_batches.append(zonefile_hashes[i:i+100])

    for zf_batch in zf_batches:
        zf_payload = None
        try:
            zf_payload = blockstack_get_zonefiles( peer_hostport, zf_batch, timeout=timeout, my_hostport=my_hostport, proxy=rpc )

        except (socket.timeout, socket.gaierror, socket.herror, socket.error), se:
            atlas_log_socket_error( "get_zonefiles(%s)" % peer_hostport, peer_hostport, se)

        except Exception, e:
            if os.environ.get("BLOCKSTACK_DEBUG") is not None:
                log.exception(e)

            log.error("Invalid zonefile data from %s" % peer_hostport)

        if zf_payload is None:
            log.error("Failed to fetch zonefile data from %s" % peer_hostport)
            atlas_peer_update_health( peer_hostport, False, peer_table=peer_table )
            
            zonefile_datas = None
            break

        if 'error' in zf_payload.keys():
            log.error("Failed to fetch zonefile data from %s: %s" % (peer_hostport, zf_payload['error']))
            atlas_peer_update_health( peer_hostport, False, peer_table=peer_table )

            zonefile_datas = None
            break

        # success!
        zonefile_datas.update( zf_payload['zonefiles'] )

    atlas_peer_update_health( peer_hostport, True, peer_table=peer_table )
    return zonefile_datas
477309
def atlasdb_sync_zonefiles( db, start_block, zonefile_dir, atlas_state, validate=True, end_block=None, path=None, con=None ):
    
    ret = None
    with AtlasDBOpen(con=con, path=path) as dbcon:
        ret = atlasdb_queue_zonefiles( dbcon, db, start_block, zonefile_dir, validate=validate, end_block=end_block )
        atlasdb_cache_zonefile_info( con=dbcon )

    if atlas_state:
        # it could have been the case that a zone file we already have was re-announced.
        # if so, then inform any storage listeners in the crawler thread that this has happened
        # (such as the subdomain system).
        crawler_thread = atlas_state['zonefile_crawler']
        for zfinfo in filter(lambda zfi: zfi['present'], ret):
            log.debug('Store re-discovered zonefile {} at {}'.format(zfinfo['zonefile_hash'], zfinfo['block_height']))
            crawler_thread.store_zonefile_cb(zfinfo['zonefile_hash'], zfinfo['block_height'])

    return ret
1077142
def get_zone(self, zone_name):
        
        for zone in self.get_zones():
            if zone_name == zone['name']:
                return zone

        raise RuntimeError("Unknown zone")
1117363
def _get_domain_text_of_authoritative_zone(self):
        """"""
        # We are logged in, so get the domain list
        from bs4 import BeautifulSoup
        zones_response = self.session.get(self.URLS['domain_list'])
        self._log('Zone', zones_response)
        assert zones_response.status_code == 200, \
            'Could not retrieve domain list due to a network error.'

        html = BeautifulSoup(zones_response.content, 'html.parser')
        self._log('Zone', html)
        domain_table = html.find('table', {'id': 'cp_domain_table'})
        assert domain_table is not None, 'Could not find domain table'

        # (Sub)domains can either be managed in their own zones or by the
        # zones of their parent (sub)domains. Iterate over all subdomains
        # (starting with the deepest one) and see if there is an own zone
        # for it.
        domain = self.domain or ''
        domain_text = None
        subdomains = domain.split('.')
        while True:
            domain = '.'.join(subdomains)
            LOGGER.debug('Check if %s has own zone', domain)
            domain_text = domain_table.find(string=domain)
            if domain_text is not None or len(subdomains) < 3:
                break
            subdomains.pop(0)

        # Update domain to equal the zone's domain. This is important if we are
        # handling a subdomain that has no zone of itself. If we do not do
        # this, self._relative_name will strip also a part of the subdomain
        # away.
        self.domain = domain
        assert domain_text is not None, \
            'The domain does not exist on Easyname.'
        return domain_text
75847
def subdomain_check_pending(self, subrec, atlasdb_path, cur=None):
        
        _, _, domain = is_address_subdomain(subrec.get_fqn())
        sql = 'SELECT missing FROM {} WHERE domain = ? ORDER BY parent_zonefile_index DESC LIMIT 1;'.format(self.subdomain_table)
        args = (domain,)

        cursor = None
        if cur is None:
            cursor = self.conn.cursor()
        else:
            cursor= cur

        rows = db_query_execute(cursor, sql, args)
        missing_str = ""
        try:
            rowdata = rows.fetchone()
            assert rowdata
            missing_str = rowdata['missing']
        except:
            pass

        known_missing = [int(i) for i in missing_str.split(',')] if missing_str is not None and len(missing_str) > 0 else []
        num_missing = atlasdb_get_zonefiles_missing_count_by_name(domain, indexes_exclude=known_missing, path=atlasdb_path)
        if num_missing > 0:
            log.debug("Subdomain is missing {} zone files: {}".format(num_missing, subrec))

        return num_missing > 0
1072703
def index(self, block_start, block_end):
        
        log.debug("BEGIN Processing zonefiles discovered since last re-indexing")
        t1 = time.time()
        self.index_discovered_zonefiles(block_end)
        t2 = time.time()
        log.debug("END Processing zonefiles discovered since last re-indexing ({} seconds)".format(t2 - t1))
1098920
def atlas_zonefile_find_push_peers( zonefile_hash, peer_table=None, zonefile_bits=None, con=None, path=None ):
    

    if zonefile_bits is None:
        zonefile_bits = atlasdb_get_zonefile_bits( zonefile_hash, path=path, con=con )
        if len(zonefile_bits) == 0:
            # we don't even know about it
            return []

    push_peers = []
    
    with AtlasPeerTableLocked(peer_table) as ptbl:
        for peer_hostport in ptbl.keys():
            zonefile_inv = atlas_peer_get_zonefile_inventory( peer_hostport, peer_table=ptbl )
            res = atlas_inventory_test_zonefile_bits( zonefile_inv, zonefile_bits )
            if res:
                push_peers.append( peer_hostport )

    return push_peers