Make remote python packages available for local use.
488995
def get_package_sources(self):
        """"""
        # Checkout local modules
        for config in self.sources.get('local', []):
            self.fetch_local_package(config=config)
        # Checkout S3 repositories specified in config
        for config in self.sources.get('s3', []):
            self.fetch_s3_package(config=config)
        # Checkout git repositories specified in config
        for config in self.sources.get('git', []):
            self.fetch_git_package(config=config)
1336703
def install_python_module_locally(name):
    
    with settings(hide('warnings', 'running', 'stdout', 'stderr'),
                  warn_only=False, capture=True):
        local('pip --quiet install %s' % name)
562520
def execute_locally(self):
        """"""
        # Make script file #
        self.make_script()
        # Do it #
        with open(self.kwargs['out_file'], 'w') as handle:
            sh.python(self.script_path, _out=handle, _err=handle)
741319
def fetch_local_package(self, config):
        
        # Update sys.path & merge in remote configs (if necessary)
        self.update_paths_and_config(config=config,
                                     pkg_dir_name=config['source'],
                                     pkg_cache_dir=os.getcwd())
950150
def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear):
    """"""
    if sys.executable.startswith(bin_dir):
        print 'Please use the *system* python to run this script'
        return

    if clear:
        rmtree(lib_dir)
        ## FIXME: why not delete it?
        ## Maybe it should delete everything with #!/path/to/venv/python in it
        logger.notify('Not deleting %s', bin_dir)

    if hasattr(sys, 'real_prefix'):
        logger.notify('Using real prefix %r' % sys.real_prefix)
        prefix = sys.real_prefix
    else:
        prefix = sys.prefix
    mkdir(lib_dir)
    fix_lib64(lib_dir)
    stdlib_dirs = [os.path.dirname(os.__file__)]
    if sys.platform == 'win32':
        stdlib_dirs.append(join(os.path.dirname(stdlib_dirs[0]), 'DLLs'))
    elif sys.platform == 'darwin':
        stdlib_dirs.append(join(stdlib_dirs[0], 'site-packages'))
    for stdlib_dir in stdlib_dirs:
        if not os.path.isdir(stdlib_dir):
            continue
        if hasattr(os, 'symlink'):
            logger.info('Symlinking Python bootstrap modules')
        else:
            logger.info('Copying Python bootstrap modules')
        logger.indent += 2
        try:
            for fn in os.listdir(stdlib_dir):
                if fn != 'site-packages' and os.path.splitext(fn)[0] in REQUIRED_MODULES:
                    copyfile(join(stdlib_dir, fn), join(lib_dir, fn))
        finally:
            logger.indent -= 2
    mkdir(join(lib_dir, 'site-packages'))
    writefile(join(lib_dir, 'site.py'), SITE_PY)
    writefile(join(lib_dir, 'orig-prefix.txt'), prefix)
    site_packages_filename = join(lib_dir, 'no-global-site-packages.txt')
    if not site_packages:
        writefile(site_packages_filename, '')
    else:
        if os.path.exists(site_packages_filename):
            logger.info('Deleting %s' % site_packages_filename)
            os.unlink(site_packages_filename)

    stdinc_dir = join(prefix, 'include', py_version)
    if os.path.exists(stdinc_dir):
        copyfile(stdinc_dir, inc_dir)
    else:
        logger.debug('No include dir %s' % stdinc_dir)

    if sys.exec_prefix != prefix:
        if sys.platform == 'win32':
            exec_dir = join(sys.exec_prefix, 'lib')
        elif is_jython:
            exec_dir = join(sys.exec_prefix, 'Lib')
        else:
            exec_dir = join(sys.exec_prefix, 'lib', py_version)
        for fn in os.listdir(exec_dir):
            copyfile(join(exec_dir, fn), join(lib_dir, fn))

    if is_jython:
        # Jython has either jython-dev.jar and javalib/ dir, or just
        # jython.jar
        for name in 'jython-dev.jar', 'javalib', 'jython.jar':
            src = join(prefix, name)
            if os.path.exists(src):
                copyfile(src, join(home_dir, name))
        # XXX: registry should always exist after Jython 2.5rc1
        src = join(prefix, 'registry')
        if os.path.exists(src):
            copyfile(src, join(home_dir, 'registry'), symlink=False)
        copyfile(join(prefix, 'cachedir'), join(home_dir, 'cachedir'),
                 symlink=False)

    mkdir(bin_dir)
    py_executable = join(bin_dir, os.path.basename(sys.executable))
    if 'Python.framework' in prefix:
        if re.search(r'/Python(?:-32|-64)*$', py_executable):
            # The name of the python executable is not quite what
            # we want, rename it.
            py_executable = os.path.join(
                    os.path.dirname(py_executable), 'python')

    logger.notify('New %s executable in %s', expected_exe, py_executable)
    if sys.executable != py_executable:
        ## FIXME: could I just hard link?
        executable = sys.executable
        if sys.platform == 'cygwin' and os.path.exists(executable + '.exe'):
            # Cygwin misreports sys.executable sometimes
            executable += '.exe'
            py_executable += '.exe'
            logger.info('Executable actually exists in %s' % executable)
        shutil.copyfile(executable, py_executable)
        make_exe(py_executable)
        if sys.platform == 'win32' or sys.platform == 'cygwin':
            pythonw = os.path.join(os.path.dirname(sys.executable), 'pythonw.exe')
            if os.path.exists(pythonw):
                logger.info('Also created pythonw.exe')
                shutil.copyfile(pythonw, os.path.join(os.path.dirname(py_executable), 'pythonw.exe'))

    if os.path.splitext(os.path.basename(py_executable))[0] != expected_exe:
        secondary_exe = os.path.join(os.path.dirname(py_executable),
                                     expected_exe)
        py_executable_ext = os.path.splitext(py_executable)[1]
        if py_executable_ext == '.exe':
            # python2.4 gives an extension of '.4' :P
            secondary_exe += py_executable_ext
        if os.path.exists(secondary_exe):
            logger.warn('Not overwriting existing %s script %s (you must use %s)'
                        % (expected_exe, secondary_exe, py_executable))
        else:
            logger.notify('Also creating executable in %s' % secondary_exe)
            shutil.copyfile(sys.executable, secondary_exe)
            make_exe(secondary_exe)

    if 'Python.framework' in prefix:
        logger.debug('MacOSX Python framework detected')

        # Make sure we use the the embedded interpreter inside
        # the framework, even if sys.executable points to
        # the stub executable in ${sys.prefix}/bin
        # See http://groups.google.com/group/python-virtualenv/
        #                              browse_thread/thread/17cab2f85da75951
        shutil.copy(
                os.path.join(
                    prefix, 'Resources/Python.app/Contents/MacOS/%s' % os.path.basename(sys.executable)),
                py_executable)

        # Copy the framework's dylib into the virtual
        # environment
        virtual_lib = os.path.join(home_dir, '.Python')

        if os.path.exists(virtual_lib):
            os.unlink(virtual_lib)
        copyfile(
            os.path.join(prefix, 'Python'),
            virtual_lib)

        # And then change the install_name of the copied python executable
        try:
            call_subprocess(
                ["install_name_tool", "-change",
                 os.path.join(prefix, 'Python'),
                 '@executable_path/../.Python',
                 py_executable])
        except:
            logger.fatal(
                "Could not call install_name_tool -- you must have Apple's development tools installed")
            raise

        # Some tools depend on pythonX.Y being present
        py_executable_version = '%s.%s' % (
            sys.version_info[0], sys.version_info[1])
        if not py_executable.endswith(py_executable_version):
            # symlinking pythonX.Y > python
            pth = py_executable + '%s.%s' % (
                    sys.version_info[0], sys.version_info[1])
            if os.path.exists(pth):
                os.unlink(pth)
            os.symlink('python', pth)
        else:
            # reverse symlinking python -> pythonX.Y (with --python)
            pth = join(bin_dir, 'python')
            if os.path.exists(pth):
                os.unlink(pth)
            os.symlink(os.path.basename(py_executable), pth)

    if sys.platform == 'win32' and ' ' in py_executable:
        # There's a bug with subprocess on Windows when using a first
        # argument that has a space in it.  Instead we have to quote
        # the value:
        py_executable = '"%s"' % py_executable
    cmd = [py_executable, '-c', 'import sys; print sys.prefix']
    logger.info('Testing executable with %s %s "%s"' % tuple(cmd))
    proc = subprocess.Popen(cmd,
                            stdout=subprocess.PIPE)
    proc_stdout, proc_stderr = proc.communicate()
    proc_stdout = os.path.normcase(os.path.abspath(proc_stdout.strip()))
    if proc_stdout != os.path.normcase(os.path.abspath(home_dir)):
        logger.fatal(
            'ERROR: The executable %s is not functioning' % py_executable)
        logger.fatal(
            'ERROR: It thinks sys.prefix is %r (should be %r)'
            % (proc_stdout, os.path.normcase(os.path.abspath(home_dir))))
        logger.fatal(
            'ERROR: virtualenv is not compatible with this system or executable')
        if sys.platform == 'win32':
            logger.fatal(
                'Note: some Windows users have reported this error when they installed Python for "Only this user".  The problem may be resolvable if you install Python "For all users".  (See https://bugs.launchpad.net/virtualenv/+bug/352844)')
        sys.exit(100)
    else:
        logger.info('Got sys.prefix result: %r' % proc_stdout)

    pydistutils = os.path.expanduser('~/.pydistutils.cfg')
    if os.path.exists(pydistutils):
        logger.notify('Please make sure you remove any previous custom paths from '
                      'your %s file.' % pydistutils)
    ## FIXME: really this should be calculated earlier
    return py_executable
1337047
def rsync(local_path, remote_path, exclude=None, extra_opts=None):
    """"""
    if not local_path.endswith('/'):
        local_path += '/'
    exclude = exclude or []
    exclude.extend(['*.egg-info', '*.pyc', '.git', '.gitignore',
                    '.gitmodules', '/build/', '/dist/'])
    with hide('running'):
        run("mkdir -p '{}'".format(remote_path))
        return rsync_project(
            remote_path, local_path, delete=True,
            extra_opts='-i --omit-dir-times -FF ' +
                       (extra_opts if extra_opts else ''),
            ssh_opts='-o StrictHostKeyChecking=no',
            exclude=exclude)
709965
def _file_local_or_remote(f, get_retriever):
    
    if os.path.exists(f):
        return f
    integration, config = get_retriever.integration_and_config(f)
    if integration:
        return integration.file_exists(f, config)
315759
def remote(self, *args):
        
        log_path = args[0]
        ChangeLog_txt = args[1]
        lib_path = args[2]
        PACKAGES_TXT = args[3]
        CHECKSUMS_MD5 = args[4]
        FILELIST_TXT = args[5]
        repo = args[6]

        if self.checks_logs(log_path, ChangeLog_txt):
            # remove old files
            self.file_remove(log_path, ChangeLog_txt.split("/")[-1])
            self.file_remove(lib_path, PACKAGES_TXT.split("/")[-1])
            self.file_remove(lib_path, CHECKSUMS_MD5.split("/")[-1])
            self.file_remove(lib_path, FILELIST_TXT.split("/")[-1])
            if repo == "slack":
                dirs = ["core/", "extra/", "pasture/"]
                for d in dirs:
                    self.file_remove(lib_path + d, "PACKAGES.TXT")
                    self.file_remove(lib_path + d, "CHECKSUMS.md5")
                self.down(lib_path + "core/", PACKAGES_TXT, repo)
                self.down(lib_path + "core/", CHECKSUMS_MD5, repo)
                self.down(lib_path + "extra/", self.EXTRA, repo)
                self.down(lib_path + "extra/", self.EXT_CHECKSUMS, repo)
                if slack_ver() != "14.0":  # no pasture/ folder for 14.0 version
                    self.down(lib_path + "pasture/", self.PASTURE, repo)
                    self.down(lib_path + "pasture/", self.PAS_CHECKSUMS, repo)
            # download new files
            if repo != "slack":
                self.down(lib_path, PACKAGES_TXT, repo)
                self.down(lib_path, CHECKSUMS_MD5, repo)
            self.down(lib_path, FILELIST_TXT, repo)
            self.down(log_path, ChangeLog_txt, repo)
663604
def install(self):
        '''
        
        '''
        remote_path = os.path.join(self.venv, 'requirements.txt')
        put(self.requirements, remote_path)
        run('{pip} install -r {requirements}'.format(
            pip=self.pip(), requirements=remote_path))
677742
def harvest_openaire_projects(source=None, setspec=None):
    """"""
    loader = LocalOAIRELoader(source=source) if source \
        else RemoteOAIRELoader(setspec=setspec)
    for grant_json in loader.iter_grants():
        register_grant.delay(grant_json)
573517
def do_setup_for_pip_local(self, repo):
        '''
        
        '''
        effective_repo_name = self.get_effective_repo_name(repo)
        self.abort_on_nonexisting_repo(
            effective_repo_name, 'setup_for_pip_local'
        )

        self.network.setup_for_pip_local(effective_repo_name)
1073076
def _setup_packages(self, sc):
        
        packages = self.py_packages
        if not packages:
            return
        for package in packages:
            mod = importlib.import_module(package)
            try:
                mod_path = mod.__path__[0]
            except AttributeError:
                mod_path = mod.__file__
            tar_path = os.path.join(self.run_path, package + '.tar.gz')
            tar = tarfile.open(tar_path, "w:gz")
            tar.add(mod_path, os.path.basename(mod_path))
            tar.close()
            sc.addPyFile(tar_path)
125740
def _do_download(version, download_base, to_dir, download_delay):
    """"""
    py_desig = 'py{sys.version_info[0]}.{sys.version_info[1]}'.format(sys=sys)
    tp = 'setuptools-{version}-{py_desig}.egg'
    egg = os.path.join(to_dir, tp.format(**locals()))
    if not os.path.exists(egg):
        archive = download_setuptools(version, download_base,
            to_dir, download_delay)
        _build_egg(egg, archive, to_dir)
    sys.path.insert(0, egg)

    # Remove previously-imported pkg_resources if present (see
    # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details).
    if 'pkg_resources' in sys.modules:
        _unload_pkg_resources()

    import setuptools
    setuptools.bootstrap_install_from = egg
258494
def check_dependencies_remote(args):
    
    cmd = [args.python, '-m', 'depends', args.requirement]
    env = dict(PYTHONPATH=os.path.dirname(__file__))
    return subprocess.check_call(cmd, env=env)
659693
function local(rootFolder) {
    // Construct name to remote id map
    const remote = path.join(rootFolder, constants.META_DIR, constants.META_REMOTE);
    const remoteSource = fs.readJsonSync(remote, 'utf8');
    const destination = constants.META_LOCAL;
    return pack(rootFolder, destination);
}
265398
def deploy_local(self, dotfiles, target_root=None):
        """"""
        if target_root is None:
            target_root = self.args.path

        for source_path, target_path in dotfiles.items():
            source_path = path.join(self.source, source_path)
            target_path = path.join(target_root, target_path)

            if path.isfile(target_path) or path.islink(target_path):
                self.log.debug('Removing existing file at %s', target_path)
                os.unlink(target_path)

            elif path.isdir(target_path):
                self.log.debug('Removing existing dir at %s', target_path)
                shutil.rmtree(target_path)

            parent_dir = path.dirname(target_path)
            if not path.isdir(parent_dir):
                self.log.debug('Creating parent dir %s', parent_dir)
                os.makedirs(parent_dir)

            if self.args.copy:
                if path.isdir(source_path):
                    self.log.debug('Copying file %s to %s',
                                   source_path, target_path)
                    shutil.copytree(source_path, target_path)
                else:
                    self.log.debug('Copying dir %s to %s',
                                   source_path, target_path)
                    shutil.copy(source_path, target_path)

            else:
                self.log.debug('Symlinking %s -> %s', target_path, source_path)
                os.symlink(source_path, target_path)
872367
def other_dependencies(ctx, server, environment):
    """"""
    if 'extra_packages' in ctx.releaser:
        server = server.lower()
        extra_pkgs = []
        if server in ["local"]:
            if 'local' in ctx.releaser.extra_packages:
                extra_pkgs.extend(ctx.releaser.extra_packages.local)
        elif server in ["testpypi", "pypitest"]:
            # these are packages not available on the test server, so install them
            # off the regular pypi server
            if 'test' in ctx.releaser.extra_packages and \
                                ctx.releaser.extra_packages.test is not None:
                extra_pkgs.extend(ctx.releaser.extra_packages.test)
        elif server in ["pypi"]:
            if 'pypi' in ctx.releaser.extra_packages and \
                                ctx.releaser.extra_packages.pypi is not None:
                extra_pkgs.extend(ctx.releaser.extra_packages.pypi)
        else:
            print("** Nothing more to install **")

        if extra_pkgs:
            print('** Other Dependencies, based on server', server, '**')

        for pkg in extra_pkgs:
            result = invoke.run('env{0}{1}{0}Scripts{0}pip{2} install {3}'
                                .format(os.sep, environment, PIP_EXT, pkg),
                                hide=True)
            if result.ok:
                print('{}[{}GOOD{}] Installed {}'.format("", GOOD_COLOR,
                                                         RESET_COLOR, pkg))
            else:
                print('{}[{}ERROR{}] Something broke trying to install '
                      'package: {}'.format("", ERROR_COLOR, RESET_COLOR, pkg))
                print(result.stderr)
                sys.exit(1)