Performs a breadth-first crawl of all possible routes from the
    starting path. Will only visit a URL once, even if it is referenced
    multiple times in a plugin. Requires user interaction in between each
    fetch.
555336
def crawl(plugin):
    '''
    '''
    # TODO: use OrderedSet?
    paths_visited = set()
    paths_to_visit = set(item.get_path() for item in once(plugin))

    while paths_to_visit and continue_or_quit():
        path = paths_to_visit.pop()
        paths_visited.add(path)

        # Run the new listitem
        patch_plugin(plugin, path)
        new_paths = set(item.get_path() for item in once(plugin))

        # Filter new items by checking against urls_visited and
        # urls_tovisit
        paths_to_visit.update(path for path in new_paths
                              if path not in paths_visited)
158209
def next(self):
        
        next = request.args.get('next')
        if next is None:
            params = self.default_redirect_params
            next = url_for(self.default_redirect_endpoint, **params)
        return next
134773
def increase_route_count(self, crawled_request):
        

        for route in self.__routing_options.routes:
            if re.compile(route).match(crawled_request.url):
                count_key = str(route) + crawled_request.method
                
                if count_key in self.__routing_count.keys():
                    self.__routing_count[count_key] += 1
                else:
                    self.__routing_count[count_key] = 1

                break
800263
def each_vertex_breadth_first(start_vertex, walk_method)
      remaining = [start_vertex]
      remaining.each_with_object([]) do |vertex, history|
        history << vertex
        yield vertex
        send(walk_method, vertex).each do |other|
          remaining << other unless history.include?(other)
        end
      end
    end
98695
def run
      loop do
        link, referer, depth = @link_queue.deq

        break if link == :END

        @http.fetch_pages(link, referer, depth).each { |page| @page_queue << page }

        delay
      end
    end
615579
def get_all_paths_from(self, start, seen=None):
        '''
        
        '''
        if seen is None:
            seen = frozenset()
        results = [(0, (start, ))]
        if start in seen or start not in self.edges:
            return results
        seen = seen | frozenset((start,))
        for node, edge_weight in self.edges[start].items():
            for subpath_weight, subpath in self.get_all_paths_from(node, seen):
                total_weight = edge_weight + subpath_weight
                full_path = (start, ) + subpath
                results.append((total_weight, full_path))
        return tuple(results)
999871
def iter(self):
        

        page = 1
        fetch_all = True
        url = "{}/{}".format(__endpoint__, self.type.RESOURCE)

        if 'page' in self.params:
            page = self.params['page']
            fetch_all = False

        response = RestClient.get(url, self.params)[self.type.RESOURCE]
        while len(response):
            for item in response:
                yield self.type(item)

            if not fetch_all:
                break
            else:
                page += 1
                self.where(page=page)
            response = RestClient.get(url, self.params)[self.type.RESOURCE]
1071482
def all(self, endpoint, *args, **kwargs):
        
        # 1. Initialize the pagination parameters.
        kwargs.setdefault('params', {})['offset'] = 0
        kwargs.setdefault('params', {})['limit'] = self.limit
        kwargs['__method__'] = 'get'
        # 2. Create an initial paginated request.
        payload = self.request(endpoint, *args, **kwargs)
        has_next = payload.get('result', {}).setdefault(
            'meta', {'next': None}
        )['next']
        # 3. Loop until the end
        while has_next:
            # 4. Increment the offset
            kwargs['params']['offset'] += self.limit
            # 5. Query again
            _payload = self.request(endpoint, *args, **kwargs)
            # 6. Add the paginated data to the global one
            payload['result']['data'].extend(_payload['result']['data'])
            # 7. Compute has_next
            has_next = _payload['result']['meta']['next']
        del payload['result']['meta']
        return payload
499374
public static void forward(
		String contextRelativePath,
		RequestDispatcher dispatcher,
		HttpServletRequest request,
		HttpServletResponse response,
		Map<String,?> args
	) throws ServletException, IOException {
		// Track original page when first accessed
		final String oldOriginal = getOriginalPage(request);
		try {
			// Set original request path if not already set
			if(oldOriginal == null) {
				setOriginalPage(request, request.getServletPath());
			}
			// Keep old dispatch page to restore
			final String oldDispatchPage = getDispatchedPage(request);
			try {
				// Store as new relative path source
				setDispatchedPage(request, contextRelativePath);
				// Keep old arguments to restore
				final Object oldArgs = request.getAttribute(Dispatcher.ARG_MAP_REQUEST_ATTRIBUTE_NAME);
				try {
					// Set new arguments
					request.setAttribute(
						Dispatcher.ARG_MAP_REQUEST_ATTRIBUTE_NAME,
						args==null ? null : AoCollections.optimalUnmodifiableMap(args)
					);
					// Perform dispatch
					dispatcher.forward(request, response);
				} finally {
					// Restore any previous args
					request.setAttribute(Dispatcher.ARG_MAP_REQUEST_ATTRIBUTE_NAME, oldArgs);
				}
			} finally {
				setDispatchedPage(request, oldDispatchPage);
			}
		} finally {
			if(oldOriginal == null) {
				setOriginalPage(request, null);
			}
		}
	}
600792
def walk(start:list, graphs:iter) -> iter:
    
    walked = set([start])
    stack = [start]
    while len(stack) > 0:
        *stack, curr = stack
        yield curr
        succs = it.chain.from_iterable(graph.get(curr, ()) for graph in graphs)
        for succ in succs:
            if succ not in walked:
                walked.add(curr)
                stack.append(succ)
15616
def keys_breadth_first(self, include_dicts=False):
        """"""
        namespaces = []
        for key in self._key_order:
            if isinstance(getattr(self, key), DotDict):
                namespaces.append(key)
                if include_dicts:
                    yield key
            else:
                yield key
        for a_namespace in namespaces:
            for key in self[a_namespace].keys_breadth_first(include_dicts):
                yield '%s.%s' % (a_namespace, key)
1140128
def first_page(self):
        
        try:
            start_url = self.start_url
        except AttributeError:
            raise PageError("Page %r is already the first page" % self)
        return self.page_for_url(start_url)
680296
def graph_route(self, request):
    """"""
    run = request.args.get('run')
    tag = request.args.get('tag', '')
    conceptual_arg = request.args.get('conceptual', False)
    is_conceptual = True if conceptual_arg == 'true' else False

    if run is None:
      return http_util.Respond(
          request, 'query parameter "run" is required', 'text/plain', 400)

    limit_attr_size = request.args.get('limit_attr_size', None)
    if limit_attr_size is not None:
      try:
        limit_attr_size = int(limit_attr_size)
      except ValueError:
        return http_util.Respond(
            request, 'query parameter `limit_attr_size` must be an integer',
            'text/plain', 400)

    large_attrs_key = request.args.get('large_attrs_key', None)

    try:
      result = self.graph_impl(run, tag, is_conceptual, limit_attr_size, large_attrs_key)
    except ValueError as e:
      return http_util.Respond(request, e.message, 'text/plain', code=400)
    else:
      if result is not None:
        (body, mime_type) = result  # pylint: disable=unpacking-non-sequence
        return http_util.Respond(request, body, mime_type)
      else:
        return http_util.Respond(request, '404 Not Found', 'text/plain',
                                 code=404)
123416
def route(self, path):
        # type: (str)->Tuple[Any, Callable]
        
        logging.getLogger(__name__).debug("Routing path '%s'.", path)

        cls = None
        for strategy in self._strategies:
            if strategy.can_route(path):
                cls = strategy.route(path)
                break

        if cls is None:
            raise RoutingError(path)

        return self._create_result(cls)
1187634
def crawl(self, urls, name='crawl', api='analyze', **kwargs):
        
        # If multiple seed URLs are specified, join with whitespace.
        if isinstance(urls, list):
            urls = ' '.join(urls)
        url = self.endpoint('crawl')
        process_url = self.endpoint(api)
        params = {
            'token': self._token,
            'seeds': urls,
            'name': name,
            'apiUrl': process_url,
        }

        # Add any additional named parameters as accepted by Crawlbot
        params['maxToCrawl'] = 10
        params.update(kwargs)

        self._get(url, params=params)

        return Job(self._token, name, self._version)
652898
function () {
        var rootURL = this._getRootURL();
        var path = this._getPath();
        var hash = this._getHash();
        var query = this._getQuery();
        var rootURLIndex = path.indexOf(rootURL);
        var routeHash, hashParts;

        Ember.assert('Path ' + path + ' does not start with the provided rootURL ' + rootURL, rootURLIndex === 0);

        // By convention, Ember.js routes using HashLocation are required to start
        // with `#/`. Anything else should NOT be considered a route and should
        // be passed straight through, without transformation.
        if (hash.substr(0, 2) === '#/') {
          // There could be extra hash segments after the route
          hashParts = hash.substr(1).split('#');
          // The first one is always the route url
          routeHash = hashParts.shift();

          // If the path already has a trailing slash, remove the one
          // from the hashed route so we don't double up.
          if (path.slice(-1) === '/') {
              routeHash = routeHash.substr(1);
          }

          // This is the "expected" final order
          path += routeHash;
          path += query;

          if (hashParts.length) {
            path += '#' + hashParts.join('#');
          }
        } else {
          path += query;
          path += hash;
        }

        return path;
      }
580370
public CrawlerResult getUrls(CrawlerConfiguration configuration) {

    final Map<String, String> requestHeaders = configuration.getRequestHeadersMap();
    final HTMLPageResponse resp =
        verifyInput(configuration.getStartUrl(), configuration.getOnlyOnPath(), requestHeaders);

    int level = 0;

    final Set<CrawlerURL> allUrls = new LinkedHashSet<CrawlerURL>();
    final Set<HTMLPageResponse> verifiedUrls = new LinkedHashSet<HTMLPageResponse>();
    final Set<HTMLPageResponse> nonWorkingResponses = new LinkedHashSet<HTMLPageResponse>();

    verifiedUrls.add(resp);

    final String host = resp.getPageUrl().getHost();

    if (configuration.getMaxLevels() > 0) {

      // set the start url
      Set<CrawlerURL> nextToFetch = new LinkedHashSet<CrawlerURL>();
      nextToFetch.add(resp.getPageUrl());

      while (level < configuration.getMaxLevels()) {

        final Map<Future<HTMLPageResponse>, CrawlerURL> futures =
            new HashMap<Future<HTMLPageResponse>, CrawlerURL>(nextToFetch.size());

        for (CrawlerURL testURL : nextToFetch) {
          futures.put(service.submit(new HTMLPageResponseCallable(testURL, responseFetcher, true,
              requestHeaders, false)), testURL);
        }

        nextToFetch =
            fetchNextLevelLinks(futures, allUrls, nonWorkingResponses, verifiedUrls, host,
                configuration.getOnlyOnPath(), configuration.getNotOnPath());
        level++;
      }
    } else {
      allUrls.add(resp.getPageUrl());
    }

    if (configuration.isVerifyUrls())
      verifyUrls(allUrls, verifiedUrls, nonWorkingResponses, requestHeaders);

    LinkedHashSet<CrawlerURL> workingUrls = new LinkedHashSet<CrawlerURL>();
    for (HTMLPageResponse workingResponses : verifiedUrls) {
      workingUrls.add(workingResponses.getPageUrl());
    }
    
    // TODO find a better fix for this
    // wow, this is a hack to fix if the first URL is redirected,
    // then we want to keep that original start url
    if (workingUrls.size() >= 1) {
      List<CrawlerURL> list = new ArrayList<CrawlerURL>(workingUrls);
      list.add(0, new CrawlerURL(configuration.getStartUrl()));
      list.remove(1);
      workingUrls.clear();
      workingUrls.addAll(list);
    }

    return new CrawlerResult(configuration.getStartUrl(), configuration.isVerifyUrls()
        ? workingUrls
        : allUrls, verifiedUrls, nonWorkingResponses);

  }