Return whether the document is likely to be a Sitemap.
435175
def is_response(cls, response):
        ''''''
        if response.body:
            if cls.is_file(response.body):
                return True
444167
public boolean isSiteUnderSite(String siteRootPath) {

        for (String siteRoot : getSiteRoots()) {
            if ((siteRootPath.length() > siteRoot.length())
                & siteRootPath.startsWith(CmsFileUtil.addTrailingSeparator(siteRoot))) {
                return true;
            }
        }
        return false;
    }
1274135
def sitemap_index(request):
    """"""
    sitemaps = []
    with db_connect() as db_connection:
        with db_connection.cursor() as cursor:
            cursor.execute("""\
                SELECT authors[1], max(revised)
                FROM latest_modules
                WHERE portal_type NOT IN ('CompositeModule', 'SubCollection')
                GROUP BY authors[1]
            """)

            for author, revised in cursor.fetchall():
                sitemaps.append(Sitemap(url=request.route_url(
                    'sitemap', from_id=author),
                    lastmod=revised))

    si = SitemapIndex(sitemaps=sitemaps)
    resp = request.response
    resp.status = '200 OK'
    resp.content_type = 'text/xml'
    resp.body = si()
    return resp
1170394
def isSupportedContent(cls, fileContent):
        """"""
        magic = bytearray(fileContent)[:4]
        return magic == p('>I', 0xfeedface) or magic == p('>I', 0xfeedfacf) or magic == p('<I', 0xfeedface) or magic == p('<I', 0xfeedfacf)
962504
def create_sitemap(app, exception):
    """"""
    if (not app.config['html_theme_options'].get('base_url', '') or
           exception is not None or
           not app.sitemap_links):
        return

    filename = app.outdir + "/sitemap.xml"
    print("Generating sitemap.xml in %s" % filename)

    root = ET.Element("urlset")
    root.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")

    for link in app.sitemap_links:
        url = ET.SubElement(root, "url")
        ET.SubElement(url, "loc").text = link

    ET.ElementTree(root).write(filename)
1163646
def is_cached(self):
        """"""
        # TODO: cache by target+hash, not per file.
        try:
            for item in self.rule.output_files:
                log.info(item)
                self.cachemgr.in_cache(item, self._metahash())
        except cache.CacheMiss:
            log.info('[%s]: Not cached.', self.address)
            return False
        else:
            log.info('[%s]: found in cache.', self.address)
            return True
256503
def add_sitemap_urls(self, rp, url_data, roboturl):
        """"""
        if not rp.sitemap_urls or not url_data.allows_simple_recursion():
            return
        for sitemap_url, line in rp.sitemap_urls:
            url_data.add_url(sitemap_url, line=line)
1239702
def is_unlikely_link(text):
    '''
    '''
    # Check for string concatenation in JavaScript
    if text[:1] in ',;+:' or text[-1:] in '.,;+:':
        return True

    # Check for unusual characters
    if re.search(r'''[\\$()'"[\]{}|<>`]''', text):
        return True

    if text[:1] == '.' \
       and not text.startswith('./') \
       and not text.startswith('../'):
        return True

    if text in ('/', '//'):
        return True

    if '//' in text and '://' not in text and not text.startswith('//'):
        return True

    # Forbid strings like mimetypes
    if text in MIMETYPES:
        return True

    tag_1, dummy, tag_2 = text.partition('.')
    if tag_1 in HTML_TAGS and tag_2 != 'html':
        return True

    # Forbid things where the first part of the path looks like a domain name
    if FIRST_PART_TLD_PATTERN.match(text):
        return True
62944
def is_cached(self, link):
        '''
        '''
        if link is None:
            return False
        elif hasattr(link, 'uri'):
            return link.uri in self.id_map
        else:
            return link in self.id_map
1257835
def is_mdgel(self):
        """"""
        # TODO: this likely reads the second page from file
        try:
            ismdgel = self.pages[0].is_mdgel or self.pages[1].is_mdgel
            if ismdgel:
                self.is_uniform = False
            return ismdgel
        except IndexError:
            return False
169799
def issamedoc(self):
        """"""
        return (self.scheme is None and self.authority is None and
                not self.path and self.query is None)
86407
def _mp_embeds_into(mp1, mp2):
    """"""
    sc_matches = []
    if mp1.monomer.name != mp2.monomer.name:
        return False
    # Check that all conditions in mp2 are met in mp1
    for site_name, site_state in mp2.site_conditions.items():
        if site_name not in mp1.site_conditions or \
           site_state != mp1.site_conditions[site_name]:
            return False
    return True
698974
def valid_result(self, r):
        '''
            
        '''
        source_mapping_elements = [elem['source_mapping']['filename_absolute'] for elem in r['elements'] if 'source_mapping' in elem]
        if r['elements'] and all((any(path in src_mapping for path in self._paths_to_filter) for src_mapping in source_mapping_elements)):
            return False
        return not r['description'] in [pr['description'] for pr in self._previous_results]
1212096
def _looks_like_libdoc_file(self, name):
        """"""
        # inefficient since we end up reading the file twice,
        # but it's fast enough for our purposes, and prevents
        # us from doing a full parse of files that are obviously
        # not libdoc files
        if name.lower().endswith(".xml"):
            with open(name, "r") as f:
                # read the first few lines; if we don't see
                # what looks like libdoc data, return false
                data = f.read(200)
                index = data.lower().find("<keywordspec ")
                if index > 0:
                    return True
        return False
808759
def _is_in_targets(self, site, targets):
        
        elems = self._get_elements(site)
        for elem in elems:
            if elem not in targets:
                return False
        return True
1325817
private static boolean hasSiteMapUrl(
		final ServletContext servletContext,
		final HttpServletRequest req,
		final HttpServletResponse resp,
		final SortedSet<View> views,
		final Book book,
		PageRef pageRef
	) throws ServletException, IOException {
		Boolean result = CapturePage.traversePagesAnyOrder(
			servletContext,
			req,
			resp,
			pageRef,
			CaptureLevel.META,
			new CapturePage.PageHandler<Boolean>() {
				@Override
				public Boolean handlePage(Page page) throws ServletException, IOException {
					// TODO: Chance for more concurrency here by view?
					for(View view : views) {
						if(
							view.getAllowRobots(servletContext, req, resp, page)
							&& view.isApplicable(servletContext, req, resp, page)
						) {
							return true;
						}
					}
					return null;
				}
			},
			new CapturePage.TraversalEdges() {
				@Override
				public Set<ChildRef> getEdges(Page page) {
					return page.getChildRefs();
				}
			},
			new CapturePage.EdgeFilter() {
				@Override
				public boolean applyEdge(PageRef childPage) {
					return book.getBookRef().equals(childPage.getBookRef());
				}
			}
		);
		assert result == null || result : "Should always be null or true";
		return result != null;
	}
1247231
def _guess_content_type(url):
    '''
    
    '''
    OCTET_STREAM = 'application/octet-stream'
    n = url.rfind('.')
    if n == -1:
        return OCTET_STREAM
    return mimetypes.types_map.get(url[n:].lower(), OCTET_STREAM)