Clones the given repos in temp directories
    :param session: requests Session instance
    :param urls: list, str URLs
    :return: tuple, (str -> directory, str -> URL)
495636
def clone_repo(session, urls):
    
    repos = []
    for url in urls:
        dir = mkdtemp()
        call = ["git", "clone", url, dir]
        subprocess.call(call)
        repos.append((dir, url))
    return repos
479381
def requests_session(self, *args, **kwargs):
        
        import requests
        auth1 = self.generate_auth()
        return requests.Session(*args, auth=auth1, **kwargs)
896360
def get(self, uri, params={}):
        '''
        '''
        return self._session.get(urljoin(Investigate.BASE_URL, uri),
            params=params, headers=self._auth_header, proxies=self.proxies
        )
163597
def _build_url_rewriter(cls, session: AppSession):
        ''''''
        if session.args.escaped_fragment or session.args.strip_session_id:
            return session.factory.new(
                'URLRewriter',
                hash_fragment=session.args.escaped_fragment,
                session_id=session.args.strip_session_id
            )
677929
def clonetopath(self, dest):
        
        raise Exception(
            "%s.%s needs to implement @classmethod .clonetopath(dest)" % (
                self.__class__.__module__, self.__class__.__name__))
257247
def git_clone(git_url, path):
    """"""
    if os.path.exists(os.path.join(path, '.git')):
        # get rid of local repo if it already exists
        shutil.rmtree(path)

    os.makedirs(path, exist_ok=True)
    print('Start cloning from {}…'.format(git_url))
    git_proc = subprocess.Popen(['git', 'clone', git_url, path],
                                stderr=subprocess.PIPE,
                                stdout=subprocess.PIPE,
                                env={'GIT_TERMINAL_PROMPT': '0'})

    try:
        stdoutmsg, stderrmsg = git_proc.communicate(timeout=120)
    except subprocess.TimeoutExpired:
        git_proc.kill()
        stderrmsg = b'Timed out.'

    if git_proc.returncode == 0:
        print('Cloned {}.'.format(git_url))
    else:
        print('Error cloning from {}:\n{}'.format(git_url,
                                                  stderrmsg.decode('utf-8')))
1131628
public static Dir fromURL(final URL url, final UrlType... urlTypes) {
        return fromURL(url, Lists.<UrlType>newArrayList(urlTypes));
    }
1366828
def do_session(self):
        """"""
        def filenames():
            'Helper'
            for arg in self.args:
                if os.path.isdir(arg):
                    for name in glob.glob(os.path.join(arg, '*.torrent.rtorrent')):
                        yield name
                elif arg == '@-':
                    for line in sys.stdin.read().splitlines():
                        if line.strip():
                            yield line.strip()
                elif arg.startswith('@'):
                    if not os.path.isfile(arg[1:]):
                        self.parser.error("File not found (or not a file): {}".format(arg[1:]))
                    with io.open(arg[1:], encoding='utf-8') as handle:
                        for line in handle:
                            if line.strip():
                                yield line.strip()
                else:
                    yield arg

        proxy = self.open()
        for filename in filenames():
            # Check filename and extract infohash
            self.LOG.debug("Reading '%s'...", filename)
            match = re.match(r'(?:.+?[-._])?([a-fA-F0-9]{40})(?:[-._].+?)?\.torrent\.rtorrent',
                             os.path.basename(filename))
            if not match:
                self.LOG.warn("Skipping badly named session file '%s'...", filename)
                continue
            infohash = match.group(1)

            # Read bencoded data
            try:
                with open(filename, 'rb') as handle:
                    raw_data = handle.read()
                data = Bunch(bencode.bdecode(raw_data))
            except EnvironmentError as exc:
                self.LOG.warn("Can't read '%s' (%s)" % (
                    filename, str(exc).replace(": '%s'" % filename, ""),
                ))
                continue

            ##print(infohash, '=', repr(data))
            if 'state_changed' not in data:
                self.LOG.warn("Skipping invalid session file '%s'...", filename)
                continue

            # Restore metadata
            was_active = proxy.d.is_active(infohash)
            proxy.d.ignore_commands.set(infohash, data.ignore_commands)
            proxy.d.priority.set(infohash, data.priority)

            if proxy.d.throttle_name(infohash) != data.throttle_name:
                proxy.d.pause(infohash)
                proxy.d.throttle_name.set(infohash, data.throttle_name)

            if proxy.d.directory(infohash) != data.directory:
                proxy.d.stop(infohash)
                proxy.d.directory_base.set(infohash, data.directory)

            for i in range(5):
                key = 'custom%d' % (i + 1)
                getattr(proxy.d, key).set(infohash, data[key])

            for key, val in data.custom.items():
                proxy.d.custom.set(infohash, key, val)

            for name in data.views:
                try:
                    proxy.view.set_visible(infohash, name)
                except xmlrpclib.Fault as exc:
                    if 'Could not find view' not in str(exc):
                        raise

            if was_active and not proxy.d.is_active(infohash):
                (proxy.d.resume if proxy.d.is_open(infohash) else proxy.d.start)(infohash)
            proxy.d.save_full_session(infohash)

            """ TODO:
                NO public "set" command! 'timestamp.finished': 1503012786,
                NO public "set" command! 'timestamp.started': 1503012784,
                NO public "set" command! 'total_uploaded': 0,
            """
67668
def clone(gandi, name, vhost, directory, origin):
    """"""
    if vhost != 'default':
        directory = vhost
    else:
        directory = name if not directory else directory

    return gandi.paas.clone(name, vhost, directory, origin)
763500
def clone(self, substitutions, commit=True, **kwargs):
        
        return self.store.clone(substitutions, **kwargs)
939359
def _make_session():
        

        sess = requests.Session()
        sess.mount('http://', requests.adapters.HTTPAdapter(max_retries=False))
        sess.mount('https://', requests.adapters.HTTPAdapter(max_retries=False))

        return sess
1373475
def clone_from(repo_url, repo_dir):
    """"""
    repo_url = _fix_repo_url(repo_url)
    LOG.info("Cloning %s into %s." % (repo_url, repo_dir))
    cmd = GIT_CLONE_CMD.format(repo_url, repo_dir)
    resp = envoy.run(cmd)
    if resp.status_code != 0:
        LOG.error("Cloned failed: %s" % resp.std_err)
        raise GitException(resp.std_err)
    LOG.info("Clone successful.")
555634
def urlopen(url, method=None, params=None, data=None, json=None,
            headers=None, allow_redirects=False, timeout=30,
            verify_ssl=True, user_agent=None):
    
    if user_agent is not None:
        warnings.warn('user_agent is no longer used with safe_urlopen')

    session = build_session()

    kwargs = {}

    if json:
        kwargs['json'] = json
        if not headers:
            headers = {}
        headers.setdefault('Content-Type', 'application/json')

    if data:
        kwargs['data'] = data

    if params:
        kwargs['params'] = params

    if headers:
        kwargs['headers'] = headers

    if method is None:
        method = 'POST' if (data or json) else 'GET'

    try:
        response = session.request(
            method=method,
            url=url,
            allow_redirects=allow_redirects,
            timeout=timeout,
            verify=verify_ssl,
            **kwargs
        )
    # Our version of requests does not transform ZeroReturnError into an
    # appropriately generically catchable exception
    except ZeroReturnError as exc:
        import sys
        exc_tb = sys.exc_info()[2]
        six.reraise(SSLError, exc, exc_tb)
        del exc_tb

    # requests' attempts to use chardet internally when no encoding is found
    # and we want to avoid that slow behavior
    if not response.encoding:
        response.encoding = 'utf-8'

    return response
917992
def clone_with_git(repo_uri, dest_path):
    
    log.info('Cloning git repo %s to %s', repo_uri, dest_path)
    git.Repo.clone_from(repo_uri, dest_path, depth=1)
123959
def urljoin(*urls):
    
    return reduce(urlparse.urljoin, [u.strip('/')+'/' for u in urls if u.strip('/')], '').rstrip('/')
230282
def clone_repo(self):
        """"""
        tempdir_path = tempfile.mkdtemp()

        if self.args.git:
            self.log.debug('Cloning git source repository from %s to %s',
                           self.source, tempdir_path)
            self.sh('git clone', self.source, tempdir_path)

        else:
            raise NotImplementedError('Unknown repo type')

        self.source = tempdir_path
607972
def resolve_url(url, directory=None, permissions=None):
    

    u = urlparse(url)

    # create the name of the destination file
    if directory is None:
        directory = os.getcwd()
    filename = os.path.join(directory,os.path.basename(u.path))

    if u.scheme == '' or u.scheme == 'file':
        # for regular files, make a direct copy
        if os.path.isfile(u.path):
            if os.path.isfile(filename):
                # check to see if src and dest are the same file
                src_inode = os.stat(u.path)[stat.ST_INO]
                dst_inode = os.stat(filename)[stat.ST_INO]
                if src_inode != dst_inode:
                    shutil.copy(u.path, filename)
            else:
                shutil.copy(u.path, filename)
        else:
            errmsg  = "Cannot open file %s from URL %s" % (u.path, url)
            raise ValueError(errmsg)

    elif u.scheme == 'http' or u.scheme == 'https':
        s = requests.Session()
        s.mount(str(u.scheme)+'://',
            requests.adapters.HTTPAdapter(max_retries=5))

        # look for an ecp cookie file and load the cookies
        cookie_dict = {}
        ecp_file = '/tmp/ecpcookie.u%d' % os.getuid()
        if os.path.isfile(ecp_file):
            cj = cookielib.MozillaCookieJar()
            cj.load(ecp_file, ignore_discard=True, ignore_expires=True)
        else:
            cj = []

        for c in cj:
            if c.domain == u.netloc:
                # load cookies for this server
                cookie_dict[c.name] = c.value
            elif u.netloc == "code.pycbc.phy.syr.edu" and \
              c.domain == "git.ligo.org":
                # handle the redirect for code.pycbc to git.ligo.org
                cookie_dict[c.name] = c.value

        r = s.get(url, cookies=cookie_dict, allow_redirects=True)
        if r.status_code != 200:
            errmsg = "Unable to download %s\nError code = %d" % (url,
                r.status_code)
            raise ValueError(errmsg)

        # if we are downloading from git.ligo.org, check that we
        # did not get redirected to the sign-in page
        if u.netloc == 'git.ligo.org' or u.netloc == 'code.pycbc.phy.syr.edu':
            # Check if we have downloaded a binary file.
            if istext(r.content):
                soup = BeautifulSoup(r.content, 'html.parser')
                desc = soup.findAll(attrs={"property":"og:url"})
                if len(desc) and \
                  desc[0]['content'] == 'https://git.ligo.org/users/sign_in':
                    raise ValueError(ecp_cookie_error.format(url))

        output_fp = open(filename, 'w')
        output_fp.write(r.content)
        output_fp.close()

    else:
        # TODO: We could support other schemes such as gsiftp by
        # calling out to globus-url-copy
        errmsg  = "Unknown URL scheme: %s\n" % (u.scheme)
        errmsg += "Currently supported are: file, http, and https."
        raise ValueError(errmsg)

    if not os.path.isfile(filename):
        errmsg = "Error trying to create file %s from %s" % (filename,url)
        raise ValueError(errmsg)

    if permissions:
        if os.access(filename, os.W_OK):
            os.chmod(filename, permissions)
        else:
            # check that the file has at least the permissions requested
            s = os.stat(filename)[stat.ST_MODE]
            if (s & permissions) != permissions:
                errmsg = "Could not change permissions on %s (read-only)" % url
                raise ValueError(errmsg)

    return filename