Add PySpark to the library path based on the value of SPARK_HOME.
545394
def add_pyspark_path():
    

    try:
        spark_home = os.environ['SPARK_HOME']

        sys.path.append(os.path.join(spark_home, 'python'))
        py4j_src_zip = glob(os.path.join(spark_home, 'python',
                                         'lib', 'py4j-*-src.zip'))
        if len(py4j_src_zip) == 0:
            raise ValueError('py4j source archive not found in %s'
                             % os.path.join(spark_home, 'python', 'lib'))
        else:
            py4j_src_zip = sorted(py4j_src_zip)[::-1]
            sys.path.append(py4j_src_zip[0])
    except KeyError:
        logging.error("""SPARK_HOME was not set. please set it. e.g.
          SPARK_HOME='/home/...' ./bin/pyspark [program]""")
        exit(-1)
    except ValueError as e:
        logging.error(str(e))
        exit(-1)
621917
def _add_to_submit_args(s):
    """"""
    new_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "") + (" %s" % s)
    os.environ["PYSPARK_SUBMIT_ARGS"] = new_args
    return new_args
143860
def libpath(self):
        
        from os import path
        return path.join(self.dirpath, self.libname)
963078
def path_locations(home_dir):
    """"""
    home_dir = os.path.abspath(home_dir)
    # XXX: We'd use distutils.sysconfig.get_python_inc/lib but its
    # prefix arg is broken: http://bugs.python.org/issue3386
    if is_win:
        # Windows has lots of problems with executables with spaces in
        # the name; this function will remove them (using the ~1
        # format):
        mkdir(home_dir)
        if ' ' in home_dir:
            import ctypes
            GetShortPathName = ctypes.windll.kernel32.GetShortPathNameW
            size = max(len(home_dir)+1, 256)
            buf = ctypes.create_unicode_buffer(size)
            try:
                u = unicode
            except NameError:
                u = str
            ret = GetShortPathName(u(home_dir), buf, size)
            if not ret:
                print('Error: the path "%s" has a space in it' % home_dir)
                print('We could not determine the short pathname for it.')
                print('Exiting.')
                sys.exit(3)
            home_dir = str(buf.value)
        lib_dir = join(home_dir, 'Lib')
        inc_dir = join(home_dir, 'Include')
        bin_dir = join(home_dir, 'Scripts')
    if is_jython:
        lib_dir = join(home_dir, 'Lib')
        inc_dir = join(home_dir, 'Include')
        bin_dir = join(home_dir, 'bin')
    elif is_pypy:
        lib_dir = home_dir
        inc_dir = join(home_dir, 'include')
        bin_dir = join(home_dir, 'bin')
    elif not is_win:
        lib_dir = join(home_dir, 'lib', py_version)
        inc_dir = join(home_dir, 'include', py_version + abiflags)
        bin_dir = join(home_dir, 'bin')
    return home_dir, lib_dir, inc_dir, bin_dir
115392
def build_from_source(version, **kwargs):
        
        mvn = os.path.join(Spark.svm_version_path(version), 'build', 'mvn')
        Spark.chmod_add_excute(mvn)
        p = subprocess.Popen([mvn, '-DskipTests', 'clean', 'package'], cwd=Spark.svm_version_path(version))
        p.wait()
        return p.returncode
1354536
def add_path(self, path):
        """"""
        if not os.path.exists(path):
            raise RuntimeError('Path does not exists: %s.' % path)
        self.paths.append(path)
229674
def _on_library_name_changed(self, renderer, path, new_library_name):
        
        old_library_name = self.library_list_store[int(path)][self.KEY_STORAGE_ID]
        if old_library_name == new_library_name:
            return
        library_path = self.library_list_store[int(path)][self.VALUE_STORAGE_ID]

        library_config = self.core_config_model.get_current_config_value("LIBRARY_PATHS", use_preliminary=True,
                                                                         default={})
        del library_config[old_library_name]
        library_config[new_library_name] = library_path
        self.core_config_model.set_preliminary_config_value("LIBRARY_PATHS", library_config)
        self._select_row_by_column_value(self.view['library_tree_view'], self.library_list_store,
                                         self.KEY_STORAGE_ID, new_library_name)
1092671
def _update_python_paths(self):
        
        for path in self._config['workflows'] + self._config['libraries']:
            if os.path.isdir(os.path.abspath(path)):
                if path not in sys.path:
                    sys.path.append(path)
            else:
                raise ConfigLoadError(
                    'Workflow directory {} does not exist'.format(path))
241303
def set_spark_conf(self, key=None, value=None, d=None):
        
        if isinstance(d, dict):
            self._properties.update(d)
        elif isinstance(key, str) and isinstance(value, str):
            self._properties[key] = value
        else:
            raise TypeError("key, value must be strings")
946299
def set_spark(cls, master=None, appName=None, conf=None, hive_support=False):
        
        sess = SparkSession.builder
        if master:
            sess.master(master)
        if appName:
            sess.appName(appName)
        if conf:
            sess.config(conf=conf)
        if hive_support:
            sess.enableHiveSupport()

        cls.spark = sess.getOrCreate()
521503
def Insert(self, index, rdfpathspec=None, **kwarg):
    """"""
    if rdfpathspec is None:
      rdfpathspec = self.__class__(**kwarg)

    if index == 0:
      # Copy ourselves to a temp copy.
      nested_proto = self.__class__()
      nested_proto.SetRawData(self.GetRawData())

      # Replace ourselves with the new object.
      self.SetRawData(rdfpathspec.GetRawData())

      # Append the temp copy to the end.
      self.last.nested_path = nested_proto
    else:
      previous = self[index - 1]
      rdfpathspec.last.nested_path = previous.nested_path
      previous.nested_path = rdfpathspec
683087
def _on_library_path_changed(self, renderer, path, new_library_path):
        
        library_name = self.library_list_store[int(path)][self.KEY_STORAGE_ID]

        library_config = self.core_config_model.get_current_config_value("LIBRARY_PATHS", use_preliminary=True,
                                                                         default={})
        library_config[library_name] = new_library_path
        self.core_config_model.set_preliminary_config_value("LIBRARY_PATHS", library_config)
        self._select_row_by_column_value(self.view['library_tree_view'], self.library_list_store,
                                         self.KEY_STORAGE_ID, library_name)
268797
def add(self, value):
        """"""
        path = value if isinstance(value, Path) else Path(str(value))
        if path and path != path.parent:
            destination = self
            for part in path.parts:
                destination = destination.setdefault(part, DirectoryTree())
992570
def build(python=PYTHON):
    """"""
    clean()
    local(
        "LIBRARY_PATH={library_path} CPATH={include_path} {python} "
        "setup.py build_ext --inplace".format(
            library_path=LIBRARY_PATH,
            include_path=INCLUDE_PATH,
            python=python,
        ))
307418
def get_model_home():
    '''
    
    '''
    d = os.path.join(get_data_home(), 'nnp_models')
    if not os.path.isdir(d):
        os.makedirs(d)
    return d
197401
def install_local_library(path)
      src_path = path.realpath
      library_name = src_path.basename
      destination_path = library_path(library_name)

      # things get weird if the sketchbook contains the library.
      # check that first
      if destination_path.exist?
        uhoh = "There is already a library '#{library_name}' in the library directory"
        return destination_path if destination_path == src_path

        # maybe it's a symlink? that would be OK
        if destination_path.symlink?
          return destination_path if destination_path.readlink == src_path

          @last_msg = "#{uhoh} and it's not symlinked to #{src_path}"
          return nil
        end

        @last_msg = "#{uhoh}.  It may need to be removed manually."
        return nil
      end

      # install the library
      Host.symlink(src_path, destination_path)
      destination_path
    end
1368962
def _set_environment_variables(self):
        """"""
        cmd = []

        # special case for driver JVM properties.
        self._set_launcher_property("driver-memory", "spark.driver.memory")
        self._set_launcher_property("driver-library-path", "spark.driver.extraLibraryPath")
        self._set_launcher_property("driver-class-path", "spark.driver.extraClassPath")
        self._set_launcher_property("driver-java-options", "spark.driver.extraJavaOptions")
        self._set_launcher_property("executor-memory", "spark.executor.memory")
        self._set_launcher_property("executor-cores", "spark.executor.cores")

        for key, val in self._spark_launcher_args.items():
            if val is None:
                continue
            val = list(as_iterable(val))
            if len(val):
                if key in self._boolean_args:
                    cmd.append("--{key}".format(key=key))
                else:
                    sep = self._spark_launcher_arg_sep.get(key, ',')
                    cmd.append('--{key} {val}'.format(key=key, val=sep.join(str(x) for x in val)))

        cmd += ['pyspark-shell']
        cmd_line = ' '.join(x for x in cmd if x)
        os.environ["PYSPARK_SUBMIT_ARGS"] = cmd_line
        log.info("spark-submit arguments: %s", cmd_line)