Store url check info into the database.
421080
def log_url (self, url_data):
        
        self.writeln(u"insert into %(table)s(urlname,"
              "parentname,baseref,valid,result,warning,info,url,line,col,"
              "name,checktime,dltime,size,cached,level,modified) values ("
              "%(base_url)s,"
              "%(url_parent)s,"
              "%(base_ref)s,"
              "%(valid)d,"
              "%(result)s,"
              "%(warning)s,"
              "%(info)s,"
              "%(url)s,"
              "%(line)d,"
              "%(column)d,"
              "%(name)s,"
              "%(checktime)d,"
              "%(dltime)d,"
              "%(size)d,"
              "%(cached)d,"
              "%(level)d,"
              "%(modified)s"
              ")%(separator)s" %
              {'table': self.dbname,
               'base_url': sqlify(url_data.base_url),
               'url_parent': sqlify((url_data.parent_url)),
               'base_ref': sqlify((url_data.base_ref)),
               'valid': intify(url_data.valid),
               'result': sqlify(url_data.result),
               'warning': sqlify(os.linesep.join(x[1] for x in url_data.warnings)),
               'info': sqlify(os.linesep.join(url_data.info)),
               'url': sqlify(urlutil.url_quote(url_data.url)),
               'line': url_data.line,
               'column': url_data.column,
               'name': sqlify(url_data.name),
               'checktime': url_data.checktime,
               'dltime': url_data.dltime,
               'size': url_data.size,
               'cached': 0,
               'separator': self.separator,
               "level": url_data.level,
               "modified": sqlify(self.format_modified(url_data.modified)),
              })
        self.flush()
484229
def db(context, data):
    """"""
    table = context.params.get("table", context.crawler.name)
    params = context.params
    params["table"] = table
    _recursive_upsert(context, params, data)
9104
def log_url (self, url_data):
        """"""
        node = self.get_node(url_data)
        if node:
            self.xml_starttag(u'node', attrs={u"name": u"%d" % node["id"]})
            self.xml_tag(u"label", node["label"])
            if self.has_part("realurl"):
                self.xml_tag(u"url", node["url"])
            self.xml_starttag(u"data")
            if node["dltime"] >= 0 and self.has_part("dltime"):
                self.xml_tag(u"dltime", u"%f" % node["dltime"])
            if node["size"] >= 0 and self.has_part("dlsize"):
                self.xml_tag(u"size", u"%d" % node["size"])
            if node["checktime"] and self.has_part("checktime"):
                self.xml_tag(u"checktime", u"%f" % node["checktime"])
            if self.has_part("extern"):
                self.xml_tag(u"extern", u"%d" % node["extern"])
            self.xml_endtag(u"data")
            self.xml_endtag(u"node")
1040776
def store_stats(self):
    """"""
    stats = self._stats()

    # Write stats to user-defined json file.
    stats_json_file_name = self.get_options().stats_local_json_file
    if stats_json_file_name:
      self.write_stats_to_json(stats_json_file_name, stats)

    # Upload to remote stats db.
    stats_upload_urls = copy.copy(self.get_options().stats_upload_urls)
    timeout = self.get_options().stats_upload_timeout
    for stats_url, auth_provider in stats_upload_urls.items():
      self.post_stats(stats_url, stats, timeout=timeout, auth_provider=auth_provider)
1121847
def log_url (self, url_data):
        """"""
        self.writeln()
        if self.has_part('url'):
            self.write_url(url_data)
        if url_data.name and self.has_part('name'):
            self.write_name(url_data)
        if url_data.parent_url and self.has_part('parenturl'):
            self.write_parent(url_data)
        if url_data.base_ref and self.has_part('base'):
            self.write_base(url_data)
        if url_data.url and self.has_part('realurl'):
            self.write_real(url_data)
        if url_data.checktime and self.has_part('checktime'):
            self.write_checktime(url_data)
        if url_data.dltime >= 0 and self.has_part('dltime'):
            self.write_dltime(url_data)
        if url_data.size >= 0 and self.has_part('dlsize'):
            self.write_size(url_data)
        if url_data.info and self.has_part('info'):
            self.write_info(url_data)
        if url_data.modified and self.has_part('modified'):
            self.write_modified(url_data)
        if url_data.warnings and self.has_part('warning'):
            self.write_warning(url_data)
        if self.has_part('result'):
            self.write_result(url_data)
        self.flush()
998341
def is_valid_url(url):
    """"""
    pieces = urlparse(url)
    return all([pieces.scheme, pieces.netloc])
979568
def get(self, key=None, view=None):
        """"""
        self.set_header("Access-Control-Allow-Origin", "*")
        self.set_header("Content-Type", "application/json")
        if key is not None:
            value = {}
            value.update(self.database[key])
            if view is not None:
                # generate a context with the relevant variables
                context = {}
                context["value"] = value
                context["ctx"] = self.ctx
                result = json.dumps(getattr(views, view)(context))
            else:
                result = json.dumps(value)
        else:
            result = json.dumps(self.database.values())

        self.write(result)
131171
def post(self, uri=''):
        "'"

        # get node...
        (node, rule_kw) = node_from_uri(uri, method=request.method)

        rule_kw.update( node )
        values = rule_kw
        xhr_data = request.get_json()
        if xhr_data:
            values.update( xhr_data )
        values.update( request.form.to_dict(flat=True) )
        values.update( request.args.to_dict(flat=True) )
        values['method'] = request.method

        # Execute the sql query with the data
        _query(node['id'], **values)

        response = make_response('ok', 201)
        return response
562746
def check(self, url_data):
        """"""
        headers = []
        for name, value in url_data.headers.items():
            if name.startswith(self.prefixes):
                headers.append(name)
        if headers:
            items = [u"%s=%s" % (name.capitalize(), url_data.headers[name]) for name in headers]
            info = u"HTTP headers %s" % u", ".join(items)
            url_data.add_info(info)
354208
def _compute_page_url(self):
        """"""
        for page in self:
            base_url = self.env['ir.config_parameter'].sudo().get_param(
                'web.base.url',
                default='http://localhost:8069'
            )

            page.page_url = (
                '{}/web#db={}&id={}&view_type=form&'
                'model=document.page.history').format(
                    base_url,
                    self.env.cr.dbname,
                    page.id
                )
1350153
private void populateAccess() throws SQLException {

    PreparedStatement statement = null ;
    try {
      statement = connection.prepareStatement(
          "INSERT INTO Access(url, referrer, time)" +
          " VALUES (?, ?, ?)");

      Random random = new Random();

      int time = random.nextInt(50) + 50;

      final int PROBABILITY_PRECISION = 100; //  1 / 100 
      final int NEW_PAGE_PROBABILITY  = 15;  //  15 / 100


      //Pages in the site :
      String[] pages = {"/a", "/b", "/c", "/d", "/e", "/f", "/g", "/h", "/i", "/j"};
      //linkMatrix[i] is the array of pages(indexes) that page_i links to.  
      int[][] linkMatrix = {{1,5,7}, {0,7,4,6,}, {0,1,7,8}, {0,2,4,6,7,9}, {0,1},
          {0,3,5,9}, {0}, {0,1,3}, {0,2,6}, {0,2,6}};

      //a mini model of user browsing a la pagerank
      int currentPage = random.nextInt(pages.length); 
      String referrer = null;

      for(int i=0; i<time; i++) {

        statement.setString(1, pages[currentPage]);
        statement.setString(2, referrer);
        statement.setLong(3, i);
        statement.execute();

        int action = random.nextInt(PROBABILITY_PRECISION);

        //go to a new page with probability NEW_PAGE_PROBABILITY / PROBABILITY_PRECISION
        if(action < NEW_PAGE_PROBABILITY) { 
          currentPage = random.nextInt(pages.length); // a random page
          referrer = null;
        }
        else {
          referrer = pages[currentPage];
          action = random.nextInt(linkMatrix[currentPage].length);
          currentPage = linkMatrix[currentPage][action];
        }
      }
      
      connection.commit();
      
    }catch (SQLException ex) {
      connection.rollback();
      throw ex;
    } finally {
      if(statement != null) {
        statement.close();
      }
    }
  }
243413
private void generateSqlInfos() {
		try {
			String loadPathSlash = loadPath.toString().replaceAll("\\\\", "/");
			Enumeration<URL> root = Thread.currentThread().getContextClassLoader()
					.getResources(loadPathSlash);

			while (root.hasMoreElements()) {
				URI uri = root.nextElement().toURI();
				String scheme = uri.getScheme();
				if (SCHEME_FILE.equals(scheme)) {
					traverse(Paths.get(uri), detectChanges && true, false);
				} else if (SCHEME_JAR.equals(scheme)) {
					FileSystem fs = null;
					try {
						fs = FileSystems.getFileSystem(uri);
					} catch (FileSystemNotFoundException ex) {
						Map<String, String> env = new HashMap<>();
						env.put("create", "false");
						fs = FileSystems.newFileSystem(uri, env);
					}
					traverse(fs.getPath(loadPathSlash), false, false);
				}
			}
		} catch (IOException | URISyntaxException e) {
			log.error("Can't load sql files.", e);
		}
	}
1159448
def write_result (self, url_data):
        """"""
        if url_data.valid:
            self.write(u'<tr><td class="valid">')
            self.write(self.part("result"))
            self.write(u'</td><td class="valid">')
            self.write(cgi.escape(_("Valid")))
        else:
            self.write(u'<tr><td class="error">')
            self.write(self.part("result"))
            self.write(u'</td><td class="error">')
            self.write(cgi.escape(_("Error")))
        if url_data.result:
            self.write(u": "+cgi.escape(url_data.result))
        self.writeln(u"</td></tr>")
770949
def add_url(self):
        """"""
        if self.url:
            self.url_data.add_url(self.url, line=self.parser.CurrentLineNumber,
                column=self.parser.CurrentColumnNumber)
            self.url = u""
778829
def log_url (self, url_data):
        """"""
        row = []
        if self.has_part("urlname"):
            row.append(url_data.base_url)
        if self.has_part("parentname"):
            row.append(url_data.parent_url)
        if self.has_part("baseref"):
            row.append(url_data.base_ref)
        if self.has_part("result"):
            row.append(url_data.result)
        if self.has_part("warningstring"):
            row.append(self.linesep.join(x[1] for x in url_data.warnings))
        if self.has_part("infostring"):
            row.append(self.linesep.join(url_data.info))
        if self.has_part("valid"):
            row.append(url_data.valid)
        if self.has_part("url"):
            row.append(url_data.url)
        if self.has_part("line"):
            row.append(url_data.line)
        if self.has_part("column"):
            row.append(url_data.column)
        if self.has_part("name"):
            row.append(url_data.name)
        if self.has_part("dltime"):
            row.append(url_data.dltime)
        if self.has_part("dlsize"):
            row.append(url_data.size)
        if self.has_part("checktime"):
            row.append(url_data.checktime)
        if self.has_part("cached"):
            row.append(0)
        if self.has_part("level"):
            row.append(url_data.level)
        if self.has_part("modified"):
            row.append(self.format_modified(url_data.modified))
        self.writerow(map(strformat.unicode_safe, row))
        self.flush()
428283
def store_api_url(self):
        """"""
        return "%s://%s%sapi/%s/store/" % (
            self.scheme,
            self.host,
            self.path,
            self.project_id,
        )
812996
def build_url_parts (self):
        
        # check userinfo@host:port syntax
        self.userinfo, host = urllib.splituser(self.urlparts[1])
        port = urlutil.default_ports.get(self.scheme, 0)
        host, port = urlutil.splitport(host, port=port)
        if port is None:
            raise LinkCheckerError(_("URL host %(host)r has invalid port") %
                    {"host": host})
        self.port = port
        # set host lowercase
        self.host = host.lower()
        if self.scheme in scheme_requires_host:
            if not self.host:
                raise LinkCheckerError(_("URL has empty hostname"))
            self.check_obfuscated_ip()
        if not self.port or self.port == urlutil.default_ports.get(self.scheme):
            host = self.host
        else:
            host = "%s:%d" % (self.host, self.port)
        if self.userinfo:
            self.urlparts[1] = "%s@%s" % (self.userinfo, host)
        else:
            self.urlparts[1] = host
        # safe anchor for later checking
        self.anchor = self.urlparts[4]
        if self.anchor is not None:
            assert isinstance(self.anchor, unicode), repr(self.anchor)