Basic crawler plugin that can be used to make a SEO audit for one site
This is just an example that requires some customizations

@param : the crawler engine that will emit events to this plugin
336456
function Plugin() {

    this.name = "Audit-Plugin";
    this.resources = new Map();
    this.duplicateContents = new Map();
    this.inLinks = new Map();
    this.outLinks = new Map();
    this.externalLinks = new Map();
    this.unparsedLinks =  new Set();
    this.images = new Map();
    this.errors = new Set();
    this.redirects = new Map();


}
139423
function engine(url) {
            var ret;

            var parts = /^(\w+):/.exec(url);
            if (!parts) {
                d.fatal("Invalid engine URI", url);
            }
            try {
                var Engine = angular.injector(['ng', 'coa.store']).get('Engine' + parts[1].ucfirst());
                ret = new Engine(url);
            } catch(e) {
                d.fatal("Unable to instantiate engine for URI", url, e);
            }

            d('STORE', 'Creating an engine for URI', url, ':', ret);

            return ret;
        }
1339739
def _build_document_scrapers(cls, session: AppSession):
        '''
        '''
        html_parser = session.factory['HTMLParser']
        element_walker = session.factory.new('ElementWalker')

        scrapers = [
            session.factory.new(
                'HTMLScraper',
                html_parser,
                element_walker,
                followed_tags=session.args.follow_tags,
                ignored_tags=session.args.ignore_tags,
                only_relative=session.args.relative,
                robots=session.args.robots,
                encoding_override=session.args.remote_encoding,
            ),
        ]

        if 'css' in session.args.link_extractors:
            css_scraper = session.factory.new(
                'CSSScraper',
                encoding_override=session.args.remote_encoding,
            )
            scrapers.append(css_scraper)
            element_walker.css_scraper = css_scraper

        if 'javascript' in session.args.link_extractors:
            javascript_scraper = session.factory.new(
                'JavaScriptScraper',
                encoding_override=session.args.remote_encoding,
            )
            scrapers.append(javascript_scraper)
            element_walker.javascript_scraper = javascript_scraper

        if session.args.sitemaps:
            scrapers.append(session.factory.new(
                'SitemapScraper', html_parser,
                encoding_override=session.args.remote_encoding,
            ))

        return scrapers
351726
private boolean fireEvent(Eventable eventable) {
		Eventable eventToFire = eventable;
		if (eventable.getIdentification().getHow().toString().equals("xpath")
				&& eventable.getRelatedFrame().equals("")) {
			eventToFire = resolveByXpath(eventable, eventToFire);
		}
		boolean isFired = false;
		try {
			isFired = browser.fireEventAndWait(eventToFire);
		} catch (ElementNotVisibleException | NoSuchElementException e) {
			if (crawlRules.isCrawlHiddenAnchors() && eventToFire.getElement() != null
					&& "A".equals(eventToFire.getElement().getTag())) {
				isFired = visitAnchorHrefIfPossible(eventToFire);
			} else {
				LOG.debug("Ignoring invisible element {}", eventToFire.getElement());
			}
		} catch (InterruptedException e) {
			LOG.debug("Interrupted during fire event");
			interruptThread();
			return false;
		}

		LOG.debug("Event fired={} for eventable {}", isFired, eventable);

		if (isFired) {
			// Let the controller execute its specified wait operation on the browser thread safe.
			waitConditionChecker.wait(browser);
			browser.closeOtherWindows();
			return true;
		} else {
			/*
			 * Execute the OnFireEventFailedPlugins with the current crawlPath with the crawlPath
			 * removed 1 state to represent the path TO here.
			 */
			plugins.runOnFireEventFailedPlugins(context, eventable,
					crawlpath.immutableCopyWithoutLast());
			return false; // no event fired
		}
	}
1127676
@Override
	public CrawlSession call() {
		Injector injector = Guice.createInjector(new CoreModule(config));
		controller = injector.getInstance(CrawlController.class);
		CrawlSession session = controller.call();
		reason = controller.getReason();
		return session;
	}
562029
function initJQueryEvents(){
    $('#searchDiv').parent().scroll(function() {
        if($(this).scrollTop() + $(this).innerHeight() >= $(this)[0].scrollHeight - 100) {
            globalFunc.addNodesInList();
        }
    });

    $('#toggleLayout').click(function() {
        toggleLayout();
    });

    $("#sliderText").html("Context with <strong>2</strong> hops");

    $("#slider").slider({
        range: "max",
        min: 0,
        max: 10,
        value: 2,
        slide: function (event, ui){
            var strHops = " hop";
            if (ui.value > 1){
                strHops = " hops";
            }
            $("#sliderText").html("Context with <strong>" + ui.value + "</strong>" + strHops);
            NB_HOPS = ui.value;
        }
    });
}
434866
def seo_data
      if @seo_data.nil?
        Rails.configuration.seo.each do |key, value|
          regex = Regexp.new(value["regex"]).match(request.path)

          unless regex.nil?
            data = Rails.configuration.seo[key]
            fallback = data["parent"].blank? ? seo_default : seo_default.merge(Rails.configuration.seo[data["parent"]])

            unless data["model"].blank?
              response = seo_dynamic(data["model"], regex[1..(regex.size - 1)])
              data = response.nil? ? {} : response
            end

            @seo_data = fallback.merge(data)
          end
        end
      end

      @seo_data ||= seo_default
    end
313771
protected void crawl(File file, MappedKeyEngineer<K,V> engineer)
	{
		
		if (!file.exists())
		{
			Debugger.println(file + " does not exist.");
			return;
		}
		if (file.isDirectory())
		{
			File[] files = IO.listFiles(file, listPattern);

			for (int i = 0; i < files.length; i++)
			{
				//recursive
				if(!this.mustSkip(files[i]))
				crawl(files[i],engineer);
			}
		}
		else
		{
			try
			{
				engineer.construct(file.getPath(), this.constructMapToText(file.getPath()));
				crawledPaths.add(file.getPath());				
			}
			catch(NoDataFoundException e)
			{
				//print warning if found not 
				Debugger.printWarn(e);
			}
		}
	}
1008164
function crawlImage ($,result, img, callback) {
      var parentUrl = result.url;

      var link = $(img).attr('src');
      var alt = $(img).attr('alt');
      if (link) {
          var linkUri = URI.linkToURI(parentUrl, link);
          pm.crawlImage(parentUrl, linkUri, alt, function(){
            addLinkToQueue(result, parentUrl, linkUri, null, null, callback);
          });
      }
      else {
        callback();
      }
}
555336
def crawl(plugin):
    '''
    '''
    # TODO: use OrderedSet?
    paths_visited = set()
    paths_to_visit = set(item.get_path() for item in once(plugin))

    while paths_to_visit and continue_or_quit():
        path = paths_to_visit.pop()
        paths_visited.add(path)

        # Run the new listitem
        patch_plugin(plugin, path)
        new_paths = set(item.get_path() for item in once(plugin))

        # Filter new items by checking against urls_visited and
        # urls_tovisit
        paths_to_visit.update(path for path in new_paths
                              if path not in paths_visited)
1015026
def preform(url, type, params = {}, &block)
      ActiveSupport::Notifications.instrument 'Seoshop', request: type, url: url, params: params do
        if connection.in_parallel?
          block.call
        else
          block.call.body
        end
      end
    end
827365
function () {
    // Create and attach our main view
    this.mainView = new MainView({
      model: this.me,
      el: document.body
    });

    // this kicks off our backbutton tracking (browser history)
    // and will cause the first matching handler in the router
    // to fire.
    this.router.history.start({
      root: '/',
      pushState: true
    });
  }
1204403
def engine_activity(engine_id)
      xml = make_xml('EngineActivityRequest', { 'engine-id' => engine_id })
      r   = execute(xml)
      arr = []
      if r.success
        r.res.elements.each('//ScanSummary') do |scan_event|
          arr << ScanSummary.parse(scan_event)
        end
      end
      arr
    end
657409
function crawlLinks(result, $, endCallback) {

    if (! result.links){
        return endCallback();
    }

    log.debug({"url" : result.url, "step" : "analyzeHTML", "message" : "CrawlLinks"});

    async.each($('link'), function(linkTag, callback) {
        crawLink($, result, linkTag, callback);
    }, endCallback);
}
1066352
function use(plugin) {
  var self = this
  var pipeline = self.pipeline
  var duplicate = false

  if (plugin && plugin.pluginId) {
    duplicate = self.plugins.some(matches)
  }

  if (!duplicate && self.plugins.indexOf(plugin) !== -1) {
    duplicate = true
  }

  if (!duplicate) {
    self.plugins.push(plugin)
    pipeline.use(plugin)
  }

  return self

  function matches(fn) {
    return fn.pluginId === plugin.pluginId
  }
}
439825
function () {
    this._ = {};
    this.listen();

    engines(this);
    helpers.methods(this);
    lookup(this);

    // temporary.
    this.define('errors', {
      compile: {
        engine: 'cannot find an engine for: ',
        method: 'expects engines to have a compile method',
      },
      render: {
        callback: 'is async and expects a callback function: ',
        engine: 'cannot find an engine for: ',
        method: 'expects engines to have a render method',
      }
    });

    this.loaders = {};
    this.cache = {};
    this.cache.data = {};

    this.cache.context = {};
    this.views = {};

    this.define('Base', require('./lib/base'));
    this.define('Collection', require('./lib/collection'));
    this.define('Item', require('./lib/item'));
    this.define('List', require('./lib/list'));
    this.define('View', require('./lib/view'));
    this.define('Views', require('./lib/views'));

    this.viewTypes = {
      layout: [],
      renderable: [],
      partial: []
    };

    this.inflections = {};
    this.handlers(utils.methods);
    this.delegateLoaders([
      'loader',
      'resolve',
      'compose',
      'iterator',
    ]);

    // initialize iterators and loaders
    loaders.iterators(this);
    loaders.base(this);
    loaders.data(this);
  }
746033
function continuer(verb, arg) {
            var result;

            // Until V8 3.19 / Chromium 29 is released, SpiderMonkey is the only
            // engine that has a deployed base of browsers that support generators.
            // However, SM's generators use the Python-inspired semantics of
            // outdated ES6 drafts.  We would like to support ES6, but we'd also
            // like to make it possible to use generators in deployed browsers, so
            // we also support Python-style generators.  At some point we can remove
            // this block.

            if (typeof StopIteration === "undefined") {
                // ES6 Generators
                try {
                    result = generator[verb](arg);
                } catch (exception) {
                    return reject(exception);
                }
                if (result.done) {
                    return Q(result.value);
                } else {
                    return when(result.value, callback, errback);
                }
            } else {
                // SpiderMonkey Generators
                // FIXME: Remove this case when SM does ES6 generators.
                try {
                    result = generator[verb](arg);
                } catch (exception) {
                    if (isStopIteration(exception)) {
                        return Q(exception.value);
                    } else {
                        return reject(exception);
                    }
                }
                return when(result, callback, errback);
            }
        }