according the urls gen {name:'', url:''} dict.
        :param urls:
        :type urls:
        :return:
        :rtype:
974529
def gen_tasks(self, urls):
        
        tasks = []
        if isinstance(urls, list):
            for url in urls:
                dat = {
                    'url': url,
                    'name': url.split('/')[-1]
                }
                if not self.overwrite and helper.is_file_ok(dat['name']):
                    self.result['skip'] += 1
                else:
                    tasks.append(dat)
        elif isinstance(urls, dict):
            # {k: v} => k is name, v is url value
            for k, v in urls.items():
                dat = {
                    'url': v,
                    'name': k,
                }
                if not self.overwrite and helper.is_file_ok(dat['name']):
                    self.result['skip'] += 1
                else:
                    tasks.append(dat)
        else:
            raise CrawlerParamsError('urls should be list/dict')
        return tasks
69945
def normalize_urls(urls):
    """"""
    _urls = []
    if isinstance(urls, list):
        if urls:
            if isinstance(urls[0], list):
                # multiple connections (list of the lists)
                _urls = urls
            elif isinstance(urls[0], str):
                # single connections (make it list of the lists)
                _urls = [urls]
        else:
            raise RuntimeError("No target host url provided.")
    elif isinstance(urls, str):
        _urls = [[urls]]
    return _urls
246897
def gen_search_gzh_url(keyword, page=1):
        
        assert isinstance(page, int) and page > 0

        qs_dict = OrderedDict()
        qs_dict['type'] = _search_type_gzh
        qs_dict['page'] = page
        qs_dict['ie'] = 'utf8'
        qs_dict['query'] = keyword

        return 'http://weixin.sogou.com/weixin?{}'.format(urlencode(qs_dict))
1274242
def get_urlpatterns(self):
        
        urlpatterns = super().get_urlpatterns()

        conversation_urlpatterns = [
            path(
                _('topic/<str:slug>-<int:pk>/'),
                self.topic_view.as_view(),
                name='topic',
            ),
            path(_('topic/create/'), self.topic_create_view.as_view(), name='topic_create'),
            path(
                _('topic/<str:slug>-<int:pk>/update/'),
                self.topic_update_view.as_view(),
                name='topic_update',
            ),
            path(
                _('topic/<str:topic_slug>-<int:topic_pk>/post/create/'),
                self.post_create_view.as_view(),
                name='post_create',
            ),
            path(
                _('topic/<str:topic_slug>-<int:topic_pk>/<int:pk>/post/update/'),
                self.post_update_view.as_view(),
                name='post_update',
            ),
            path(
                _('topic/<str:topic_slug>-<int:topic_pk>/<int:pk>/post/delete/'),
                self.post_delete_view.as_view(),
                name='post_delete',
            ),
        ]

        urlpatterns += [
            path(
                _('forum/<str:forum_slug>-<int:forum_pk>/'),
                include(conversation_urlpatterns),
            ),
        ]

        return urlpatterns
293358
def get_url_array(self):
        
        urlarray = []
        for urlobjects in self.__json_object["base_urls"]:
            urlarray.append(urlobjects["url"])
        return urlarray
139838
def parseUrl(url):
    
    scheme, netloc, url, params, query, fragment = urllib.parse.urlparse(url)
    query_dict = {
        k: sorted(v) if len(v) > 1 else v[0]
        for k, v in list(urllib.parse.parse_qs(query).items())
    }
    return {
        'scheme': scheme,
        'netloc': netloc,
        'url': url,
        'params': params,
        'query': query_dict,
        'fragment': fragment,
    }
1238170
def urls(cls):
        
        class_name = cls.get_class_name()
        if django.VERSION >= (2, 0):
            return path('{}/'.format(class_name), include([
                path('popup/', cls.create().as_view(), name='{}_popup_create'.format(class_name)),
                path('popup/<int:pk>/', cls.update().as_view(), name='{}_popup_update'.format(class_name)),
                path('popup/delete/<int:pk>/', cls.delete().as_view(), name='{}_popup_delete'.format(class_name)),
            ]))
        else:
            return url(r'^{}/'.format(class_name), include([
                url(r'^popup/$', cls.create().as_view(), name='{}_popup_create'.format(class_name)),
                url(r'^popup/(?P<pk>\d+)/$', cls.update().as_view(), name='{}_popup_update'.format(class_name)),
                url(r'^popup/delete/(?P<pk>\d+)/$', cls.delete().as_view(), name='{}_popup_delete'.format(class_name)),
            ]))
964853
def url_for(self, *args: str, **kwargs: str) -> URL:
        """"""
        return self._resource.url_for(*args, **kwargs)
483519
def _fetch(self, urls):
        
        responses = self._request_bulk(urls)
        for response in responses:
            try:
                soup = BeautifulSoup(response.content, 'html.parser',
                                     from_encoding="iso-8859-1")
                text = soup.get_text()
            except Exception:
                text = response.text
            self.data.append(text) # Opportunistic findings
        return responses
325037
def url_for(context, __route_name, **parts):
    
    app = context['app']

    query = None
    if 'query_' in parts:
        query = parts.pop('query_')

    for key in parts:
        val = parts[key]
        if isinstance(val, str):
            # if type is inherited from str expilict cast to str makes sense
            # if type is exactly str the operation is very fast
            val = str(val)
        elif type(val) is int:
            # int inherited classes like bool are forbidden
            val = str(val)
        else:
            raise TypeError("argument value should be str or int, "
                            "got {} -> [{}] {!r}".format(key, type(val), val))
        parts[key] = val

    url = app.router[__route_name].url_for(**parts)
    if query:
        url = url.with_query(query)
    return url
586219
def generate_urls(self, first_url, last_url):
        
        first_url = first_url.split("/")
        last_url = last_url.split("/")
        if first_url[0].lower() != "http:" or last_url[0].lower() != "http:":
            raise Exception("URLs should be accessible via HTTP.")

        url_base = "/".join(first_url[:-1])
        start_index = first_url[-1].index("a")
        file_name = first_url[-1][0:start_index]
        url_base += "/" + file_name

        start = first_url[-1][start_index:]
        finish = last_url[-1][start_index:]

        if start.count(".") == 1 and finish.count(".") == 1:
            start, file_extension = start.split(".")
            finish, _ = finish.split(".")
            if len(start) != len(finish):
                raise Exception("Filenames in url should have the same length.")
            file_extension = "." + file_extension
        else:
            raise Exception("URLs does not have the same pattern.")

        alphabet = "abcdefghijklmnopqrstuvwxyz"
        product = itertools.product(alphabet, repeat=len(start))

        urls = []
        for p in product:
            urls.append([url_base + "".join(p) + file_extension])
            if "".join(p) == finish:
                break
        return urls
1357413
def fetch_urls(urls, timeout=None):
        
        threads = [threading.Thread(target=SimpleCrawler._fetch_url, args=(url, True, timeout)) for url in urls]
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()

        results = copy.deepcopy(SimpleCrawler._results)
        SimpleCrawler._results = {}
        return results
474640
def get_urls(self, **kwargs):
        
        kwargs["site"] = Site.objects.get(id=current_site_id())
        return super(DisplayableSitemap, self).get_urls(**kwargs)
525169
def _get_url(url):
        
        if isinstance(url, urlparse.ParseResult):
            return urlparse.urlunparse(url)
        else:
            return url
209647
def _build_urlmapping(urls, strict_slashes=False, **kwargs):
    

    rules = _build_rules(urls)
    return Map(rules=list(rules), strict_slashes=strict_slashes, **kwargs)
461724
def resource_urls(request):
    """"""

    url_parsed = urlparse(settings.SEARCH_URL)
    defaults = dict(
        APP_NAME=__description__,
        APP_VERSION=__version__,
        SITE_URL=settings.SITE_URL.rstrip('/'),
        SEARCH_TYPE=settings.SEARCH_TYPE,
        SEARCH_URL=settings.SEARCH_URL,
        SEARCH_IP='%s://%s:%s' % (url_parsed.scheme, url_parsed.hostname, url_parsed.port)
    )
    return defaults
244703
def add(self, urls):
        

        if isinstance(urls, (list, tuple)):
            self.urls.extend(urls)
        elif isinstance(urls, basestring):
            self.urls.append(urls)
        elif isinstance(urls, QuerySet):
            for obj in urls:
                self.urls.append(obj.get_absolute_url())
        elif hasattr(urls, 'get_absolute_url'):
            self.urls.append(urls.get_absolute_url())
        else:
            raise TypeError("Don't know how to handle %r" % urls)