Log URL data in sitemap format.
338377
def log_url (self, url_data, priority=None):
        """"""
        self.xml_starttag(u'url')
        self.xml_tag(u'loc', url_data.url)
        if url_data.modified:
            self.xml_tag(u'lastmod', self.format_modified(url_data.modified, sep="T"))
        self.xml_tag(u'changefreq', self.frequency)
        self.xml_tag(u'priority', "%.2f" % priority)
        self.xml_endtag(u'url')
        self.flush()
1224510
private function addSitemap()
    {
        $parsed = $this->parseURL($this->encode_url($this->current_word));
        if ($parsed !== false) {
            $this->sitemap[] = $this->current_word;
            $this->sitemap = array_unique($this->sitemap);
        }
    }
220933
def _log(self, s):
    """
    # TODO(tewalds): Should this be using logging.info instead? How to see them
    # outside of google infrastructure?
    sys.stderr.write(s)
    sys.stderr.flush()
1014491
def check_url(url_data, logger):
    """"""
    if url_data.has_result:
        logger.log_url(url_data.to_wire())
    else:
        cache = url_data.aggregate.result_cache
        key = url_data.cache_url
        result = cache.get_result(key)
        if result is None:
            # check
            check_start = time.time()
            try:
                url_data.check()
                do_parse = url_data.check_content()
                url_data.checktime = time.time() - check_start
                # Add result to cache
                result = url_data.to_wire()
                cache.add_result(key, result)
                for alias in url_data.aliases:
                    # redirect aliases
                    cache.add_result(alias, result)
                # parse content recursively
                # XXX this could add new warnings which should be cached.
                if do_parse:
                    parser.parse_url(url_data)
            finally:
                # close/release possible open connection
                url_data.close_connection()
        else:
            # copy data from cache and adjust it
            result = copy.copy(result)
            result.parent_url = url_data.parent_url
            result.base_ref = url_data.base_ref or u""
            result.base_url = url_data.base_url or u""
            result.line = url_data.line
            result.column = url_data.column
            result.level = url_data.recursion_level
            result.name = url_data.name
        logger.log_url(result)
77613
def parse(self, s):
        """"""
        self._sitemaps = [ ]
        self.__rulesets = [ ]
        
        if (PY_MAJOR_VERSION > 2) and (isinstance(s, bytes) or isinstance(s, bytearray)) or \
           (PY_MAJOR_VERSION == 2) and (not isinstance(s, unicode)):            
            s = s.decode("iso-8859-1")
    
        # Normalize newlines.
        s = _end_of_line_regex.sub("\n", s)
        
        lines = s.split("\n")
        
        previous_line_was_a_user_agent = False
        current_ruleset = None
        
        for line in lines:
            line = line.strip()
            
            if line and line[0] == '#':
                # "Lines containing only a comment are discarded completely, 
                # and therefore do not indicate a record boundary." (MK1994)
                pass
            else:
                # Remove comments
                i = line.find("#")
                if i != -1: line = line[:i]
        
                line = line.strip()
                
                if not line:
                    # An empty line indicates the end of a ruleset.
                    if current_ruleset and current_ruleset.is_not_empty():
                        self.__rulesets.append(current_ruleset)
                    
                    current_ruleset = None
                    previous_line_was_a_user_agent = False
                else:
                    # Each non-empty line falls into one of six categories:
                    # 1) User-agent: blah blah blah
                    # 2) Disallow: blah blah blah
                    # 3) Allow: blah blah blah
                    # 4) Crawl-delay: blah blah blah
                    # 5) Sitemap: blah blah blah
                    # 6) Everything else
                    # 1 - 5 are interesting and I find them with the regex 
                    # below. Category 6 I discard as directed by the MK1994 
                    # ("Unrecognised headers are ignored.")
                    # Note that 4 & 5 are specific to GYM2008 syntax, but 
                    # respecting them here is not a problem. They're just 
                    # additional information the the caller is free to ignore.
                    matches = _directive_regex.findall(line)
                    
                    # Categories 1 - 5 produce two matches, #6 produces none.
                    if matches:
                        field, data = matches[0]
                        field = field.lower()
                        data = _scrub_data(data)

                        # Matching "useragent" is a deviation from the 
                        # MK1994/96 which permits only "user-agent".
                        if field in ("useragent", "user-agent"):
                            if previous_line_was_a_user_agent:
                                # Add this UA to the current ruleset 
                                if current_ruleset and data:
                                    current_ruleset.add_robot_name(data)
                            else:
                                # Save the current ruleset and start a new one.
                                if current_ruleset and current_ruleset.is_not_empty():
                                    self.__rulesets.append(current_ruleset)
                                #else:
                                    # (is_not_empty() == False) ==> malformed 
                                    # robots.txt listed a UA line but provided
                                    # no name or didn't provide any rules 
                                    # for a named UA.
                                current_ruleset = _Ruleset()
                                if data: 
                                    current_ruleset.add_robot_name(data)
                            
                            previous_line_was_a_user_agent = True
                        elif field == "allow":
                            previous_line_was_a_user_agent = False
                            if current_ruleset:
                                current_ruleset.add_allow_rule(data)
                        elif field == "sitemap":
                            previous_line_was_a_user_agent = False
                            self._sitemaps.append(data)
                        elif field == "crawl-delay":
                            # Only Yahoo documents the syntax for Crawl-delay.
                            # ref: http://help.yahoo.com/l/us/yahoo/search/webcrawler/slurp-03.html
                            previous_line_was_a_user_agent = False
                            if current_ruleset:
                                try:
                                    current_ruleset.crawl_delay = float(data)
                                except ValueError:
                                    # Invalid crawl-delay -- ignore.
                                    pass
                        else:
                            # This is a disallow line
                            previous_line_was_a_user_agent = False
                            if current_ruleset:
                                current_ruleset.add_disallow_rule(data)

        if current_ruleset and current_ruleset.is_not_empty():
            self.__rulesets.append(current_ruleset)
            
        # Now that I have all the rulesets, I want to order them in a way 
        # that makes comparisons easier later. Specifically, any ruleset that 
        # contains the default user agent '*' should go at the end of the list
        # so that I only apply the default as a last resort. According to 
        # MK1994/96, there should only be one ruleset that specifies * as the 
        # user-agent, but you know how these things go.
        not_defaults = [r for r in self.__rulesets if not r.is_default()]
        defaults = [r for r in self.__rulesets if r.is_default()]

        self.__rulesets = not_defaults + defaults
1203892
def do_print (self, url_data):
        """"""
        if self.verbose:
            return True
        if self.warnings and url_data.warnings:
            return True
        return not url_data.valid
567298
public function generateSitemap()
    {
        $sitemapXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n";

        $repository = $this->em->getRepository('WebCMS\Entity\Page');
        $pages = $repository->findAll();

        foreach ($pages as $page) {
            if ($page->getParent() !== null && $page->getVisible()) {
                $sitemapXml .= "<url>\n\t<loc>".$this->getSitemapLink($page)."</loc>\n</url>\n";
            }
        }

        $sitemapXml .= '</urlset>';

        file_put_contents('./sitemap.xml', $sitemapXml);
    }
110391
def read_component_sitemap(
            self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file):
        
        if (sitemapindex_is_file):
            if (not self.is_file_uri(sitemap_uri)):
                # Attempt to map URI to local file
                remote_uri = sitemap_uri
                sitemap_uri = self.mapper.src_to_dst(remote_uri)
                self.logger.info(
                    "Mapped %s to local file %s" %
                    (remote_uri, sitemap_uri))
            else:
                # The individual sitemaps should be at a URL (scheme/server/path)
                # that the sitemapindex URL can speak authoritatively about
                if (self.check_url_authority and
                        not UrlAuthority(sitemapindex_uri).has_authority_over(sitemap_uri)):
                    raise ListBaseIndexError(
                        "The sitemapindex (%s) refers to sitemap at a location it does not have authority over (%s)" %
                        (sitemapindex_uri, sitemap_uri))
        try:
            fh = URLopener().open(sitemap_uri)
            self.num_files += 1
        except IOError as e:
            raise ListBaseIndexError(
                "Failed to load sitemap from %s listed in sitemap index %s (%s)" %
                (sitemap_uri, sitemapindex_uri, str(e)))
        # Get the Content-Length if we can (works fine for local files)
        try:
            self.content_length = int(fh.info()['Content-Length'])
            self.bytes_read += self.content_length
        except KeyError:
            # If we don't get a length then c'est la vie
            pass
        self.logger.info(
            "Reading sitemap from %s (%d bytes)" %
            (sitemap_uri, self.content_length))
        component = sitemap.parse_xml(fh=fh, sitemapindex=False)
        # Copy resources into self, check any metadata
        for r in component:
            self.resources.add(r)
1274135
def sitemap_index(request):
    """"""
    sitemaps = []
    with db_connect() as db_connection:
        with db_connection.cursor() as cursor:
            cursor.execute("""\
                SELECT authors[1], max(revised)
                FROM latest_modules
                WHERE portal_type NOT IN ('CompositeModule', 'SubCollection')
                GROUP BY authors[1]
            """)

            for author, revised in cursor.fetchall():
                sitemaps.append(Sitemap(url=request.route_url(
                    'sitemap', from_id=author),
                    lastmod=revised))

    si = SitemapIndex(sitemaps=sitemaps)
    resp = request.response
    resp.status = '200 OK'
    resp.content_type = 'text/xml'
    resp.body = si()
    return resp
853684
def log(self, metrics_dict):
        """"""
        if self.writer:
            self.write_to_file(metrics_dict)

        if self.verbose:
            self.print_to_screen(metrics_dict)
        self.reset()
869724
def log(host=None, port=None, limit=0):
    """"""
    app.config['HOST'] = first_value(host, app.config.get('HOST',None), '0.0.0.0')
    app.config['PORT'] = int(first_value(port, app.config.get('PORT',None), 5001))
    if app.config['HOST'] == "0.0.0.0": 
        host="127.0.0.1"
    else:
        host = app.config['HOST']
    port = app.config['PORT']    
    try:
        res = requests.get('http://%s:%s/api/log?limit=%s' % (host, port, limit))
        if res.status_code == 200:
            for record in json.loads(res.text):
                if record['level'] >= 30:
                    print(record['msg'], file=sys.stderr)
                else:    
                    print(record['msg'], file=sys.stdinfo)
        else:
            print('web server is not flaskserver', file=sys.stderr)
    except:
        print('web server is not flaskserver or not start', file=sys.stderr)
845660
def sitemap(request):
    """"""
    xml = Sitemap()
    author = request.matchdict.get('from_id')
    if author is not None:
        match = "AND '{}' = authors[1]".format(author)
    else:
        match = ''

    with db_connect() as db_connection:
        with db_connection.cursor() as cursor:
            cursor.execute("""SELECT
                    ident_hash(uuid, major_version, minor_version)
                        AS idver,
                    name,
                    revised
                FROM latest_modules
                WHERE portal_type NOT IN ('CompositeModule', 'SubCollection')
                {}
                ORDER BY module_ident DESC""".format(match))
            res = cursor.fetchall()
            for ident_hash, page_name, revised in res:
                #  replace punctuation with whitespace
                page_name = re.sub(NON_WORD, ' ',
                                   page_name.decode('utf-8'), re.UNICODE)
                # remove leading and trailing whitespace
                page_name = page_name.strip().encode('utf-8')
                # replace spaces with dashes
                page_name = re.sub(' +', '-',
                                   page_name.decode('utf-8'), re.UNICODE)
                url = request.route_url('content',
                                        ident_hash=ident_hash,
                                        ignore=u'/{}'.format(page_name))
                if notblocked(url):
                    xml.add_url(url, lastmod=revised)

    resp = request.response
    resp.status = '200 OK'
    resp.content_type = 'text/xml'
    resp.body = xml()
    return resp
1099560
public function UpdateSitemap()
    {
        // Get url
        $url = $this->app->url();

        // Get basePath
        $basePath = $this->app->basePath();

        // Create sitemap
        $sitemap = new Sitemap($basePath.'/sitemap.xml');

        // Get all discussions
        $discussions = Discussion::whereVisibleTo(new Guest())->get();

        // Get all users
        $users = User::whereVisibleTo(new Guest())->get();

        // Add home
        $sitemap->addItem($url, time(), Sitemap::DAILY, 0.9);

        // Add users
        foreach ($users as $user) {
            $sitemap->addItem($url.'/u/'.$user->username, time(), Sitemap::DAILY, 0.5);
        }

        // Get all tags
        if (class_exists(Tag::class)) {
            $tags = Tag::whereVisibleTo(new Guest())->get();

            // Add tags
            foreach ($tags as $tag) {
                $sitemap->addItem($url.'/t/'.$tag->slug, time(), Sitemap::DAILY, 0.9);
            }
        }

        // Get all pages
        if (class_exists(Page::class)) {
            $pages = Page::all();

            //Add pages
            foreach ($pages as $page) {
                $sitemap->addItem($url.'/p/'.$page->id.'-'.$page->slug, time(), Sitemap::DAILY, 0.5);
            }
        }

        // Add discussions
        foreach ($discussions as $discussion) {
            $sitemap->addItem($url.'/d/'.$discussion->id.'-'.$discussion->slug, strtotime($discussion->last_time), Sitemap::DAILY, 0.7);
        }

        // Write
        $sitemap->write();
    }
1242590
def log_url (self, url_data):
        """"""
        self.check_active_loggers()
        do_print = self.do_print(url_data)
        # Only send a transport object to the loggers, not the complete
        # object instance.
        for log in self.loggers:
            log.log_filter_url(url_data, do_print)
1342225
def viewlog(calc_id, host='localhost', port=8000):
    
    base_url = 'http://%s:%s/v1/calc/' % (host, port)
    start = 0
    psize = 10  # page size
    try:
        while True:
            url = base_url + '%d/log/%d:%d' % (calc_id, start, start + psize)
            rows = json.load(urlopen(url))
            for row in rows:
                print(' '.join(row))
            start += len(rows)
            time.sleep(1)
    except:
        pass
1242916
def log_status (self, checked, in_progress, queue, duration, num_urls):
        """"""
        msg = _n("%2d thread active", "%2d threads active", in_progress) % \
          in_progress
        self.write(u"%s, " % msg)
        msg = _n("%5d link queued", "%5d links queued", queue) % queue
        self.write(u"%s, " % msg)
        msg = _n("%4d link", "%4d links", checked) % checked
        self.write(u"%s" % msg)
        msg = _n("%3d URL", "%3d URLs", num_urls) % num_urls
        self.write(u" in %s checked, " % msg)
        msg = _("runtime %s") % strformat.strduration_long(duration)
        self.writeln(msg)
        self.flush()
293068
def write_url (self, url_data):
        """"""
        self.writeln(u"<tr>")
        self.writeln(u'<td class="url">%s</td>' % self.part("url"))
        self.write(u'<td class="url">')
        self.write(u"`%s'" % cgi.escape(url_data.base_url))
        self.writeln(u"</td></tr>")