Determine what dask distributed client to use.
964633
def _get_client(self, client=True):
        """"""
        client = client or None  # convert False/None to None
        if client is True and get_client is None:
            log.debug("'dask.distributed' library was not found, will "
                      "use simple serial processing.")
            client = None
        elif client is True:
            try:
                # get existing client
                client = get_client()
            except ValueError:
                log.warning("No dask distributed client was provided or found, "
                            "but distributed features were requested. Will use simple serial processing.")
                client = None
        return client
559259
def __gather_avail(self):
        '''
        
        '''
        avail = {}
        for saltenv in self._get_envs():
            avail[saltenv] = self.client.list_states(saltenv)
        return avail
543199
function(client, sinfo) {
  let proxy = client.proxies;
  return !!proxy.sys && !! proxy.sys[sinfo.serverType];
}
486394
def get_gauge(self, name=None):
        '''
        '''
        return self.get_client(name=name, class_=statsd.Gauge)
240319
def apply_array_ufunc(func, *args, dask='forbidden'):
    """"""
    if any(isinstance(arg, dask_array_type) for arg in args):
        if dask == 'forbidden':
            raise ValueError('apply_ufunc encountered a dask array on an '
                             'argument, but handling for dask arrays has not '
                             'been enabled. Either set the ``dask`` argument '
                             'or load your data into memory first with '
                             '``.load()`` or ``.compute()``')
        elif dask == 'parallelized':
            raise ValueError("cannot use dask='parallelized' for apply_ufunc "
                             'unless at least one input is an xarray object')
        elif dask == 'allowed':
            pass
        else:
            raise ValueError('unknown setting for dask array handling: {}'
                             .format(dask))
    return func(*args)
558376
def client_for(self, config_path, quiet=False, bootstrap_server=False,
                   create_client=False):
        """"""
        client = None
        abs_path = os.path.abspath(config_path)
        if abs_path in self.clients:
            client = self.clients[abs_path]
        elif create_client:
            client = self.create_client(config_path)
            if client.setup(quiet=quiet, bootstrap_server=bootstrap_server):
                self.clients[abs_path] = client
        return client
980509
def _get_client(region, key, keyid, profile):
    '''
    
    '''
    session = _get_session(region, key, keyid, profile)
    if not session:
        log.error("Failed to get datapipeline client.")
        return None

    return session.client('datapipeline')
792214
def get_lb_conn(dd_driver=None):
    '''
    
    '''
    vm_ = get_configured_provider()

    region = config.get_cloud_config_value(
        'region', vm_, __opts__
    )

    user_id = config.get_cloud_config_value(
       'user_id', vm_, __opts__
    )
    key = config.get_cloud_config_value(
       'key', vm_, __opts__
    )
    if not dd_driver:
        raise SaltCloudSystemExit(
            'Missing dimensiondata_driver for get_lb_conn method.'
        )
    return get_driver_lb(Provider_lb.DIMENSIONDATA)(user_id, key, region=region)
961691
def get_client_state(self):
        """"""
        q = '''SELECT threat_type,platform_type,threat_entry_type,client_state FROM threat_list'''
        output = {}
        with self.get_cursor() as dbc:
            dbc.execute(q)
            for h in dbc.fetchall():
                threat_type, platform_type, threat_entry_type, client_state = h
                threat_list_tuple = (threat_type, platform_type, threat_entry_type)
                output[threat_list_tuple] = client_state
        return output
863639
def cli(env):
    """"""
    mgr = SoftLayer.LoadBalancerManager(env.client)

    routing_methods = mgr.get_routing_methods()
    table = formatting.KeyValueTable(['ID', 'Name'])
    table.align['ID'] = 'l'
    table.align['Name'] = 'l'
    table.sortby = 'ID'
    for routing_method in routing_methods:
        table.add_row([routing_method['id'], routing_method['name']])

    env.fout(table)
784558
def main():
    """"""
    init_logging()

    # Get Dask client
    arlexecute.set_client(get_dask_Client())
    arlexecute.run(init_logging)

    LOG.info('Results dir = %s', RESULTS_DIR)
    LOG.info("Starting imaging-modeling")

    # Read parameters
    PARFILE = 'parameters.json'
    if len(sys.argv) > 1:
       PARFILE = sys.argv[1]
    LOG.info("JSON parameter file = %s", PARFILE)

    try: 	
       with open(PARFILE, "r") as par_file:
             jspar = json.load(par_file)       
    except AssertionError as error:
       LOG.critical('ERROR %s', error)
       return

    # Model parameters
    configuration= jspar["modeling"]["configuration"]["name"]
    num_freq_win = jspar["modeling"]["configuration"]["num_freq_win"] # 7
    num_times    = jspar["modeling"]["configuration"]["num_times"] # 11
    r_max        = jspar["modeling"]["configuration"]["r_max"] # 300.0
    fstart	 = jspar["modeling"]["configuration"]["fstart"]
    fend	 = jspar["modeling"]["configuration"]["fend"]
    timestart_pi = jspar["modeling"]["configuration"]["timestart_pi"] # -1/3
    timeend_pi   = jspar["modeling"]["configuration"]["timeend_pi"] # 1/3
    polframe     = jspar["modeling"]["configuration"]["PolarisationFrame"] # StokesI

    frequency = numpy.linspace(fstart, fend, num_freq_win)
    channel_bw = numpy.array(num_freq_win * [frequency[1] - frequency[0]]) # 0.9e8 ... 1.1e8
    times = numpy.linspace(numpy.pi * timestart_pi, numpy.pi * timeend_pi, num_times)

    phase_centre = SkyCoord(	ra     =jspar["modeling"]["phasecentre"]["RA"] * u.deg, 
				dec    =jspar["modeling"]["phasecentre"]["Dec"] * u.deg,
                            	frame  =jspar["modeling"]["phasecentre"]["frame"], 
				equinox=jspar["modeling"]["phasecentre"]["equinox"])

    # Simulate visibilities
    vis_list = simulate_arlexecute(configuration,
                                  frequency=frequency,
                                  channel_bandwidth=channel_bw,
                                  times=times,
                                  phasecentre=phase_centre,
                                  order=jspar["modeling"]["simulate"]["order"],
                                  rmax=r_max)

    LOG.info('%d elements in vis_list', len(vis_list))
    LOG.info('About to make visibility')
    vis_list = arlexecute.compute(vis_list, sync=True)
    LOG.debug('vis_list type: %s', type(vis_list))
    LOG.debug('vis_list element type: %s', type(vis_list[0]))
    try:
        export_blockvisibility_to_hdf5(vis_list,
                                       '%s/%s' % (RESULTS_DIR, jspar["files"]["vis_list"]))
    except AssertionError as error:
        LOG.critical('ERROR %s', error)
        return

    wprojection_planes = jspar["advice"]["wprojection_planes"]
    guard_band_image   = jspar["advice"]["guard_band_image"]
    delA               = jspar["advice"]["delA"]
    advice_low = advise_wide_field(vis_list[0], guard_band_image=guard_band_image,
                                   delA=delA,
                                   wprojection_planes=wprojection_planes)
    advice_high = advise_wide_field(vis_list[-1], guard_band_image=guard_band_image,
                                    delA=delA,
                                    wprojection_planes=wprojection_planes)


    vis_slices = advice_low['vis_slices']
    num_pixels = advice_high['npixels2']
    cellsize = min(advice_low['cellsize'], advice_high['cellsize'])

    # Create GLEAM model
    gleam_model = [
        arlexecute.execute(create_low_test_image_from_gleam)(
            npixel=num_pixels,
            frequency=[frequency[f]],
            channel_bandwidth=[channel_bw[f]],
            cellsize=cellsize,
            phasecentre=phase_centre,
            polarisation_frame=PolarisationFrame(polframe),
            flux_limit=jspar["modeling"]["gleam_model"]["flux_limit"], # 1.0,
            applybeam =jspar["modeling"]["gleam_model"]["applybeam"])  # True
        for f, freq in enumerate(frequency)
    ]


    LOG.info('About to make GLEAM model')
    gleam_model = arlexecute.compute(gleam_model, sync=True)
    # future_gleam_model = arlexecute.scatter(gleam_model)

    # Get predicted visibilities for GLEAM model
    LOG.info('About to run predict to get predicted visibility')
    future_vis_graph = arlexecute.scatter(vis_list)
    predicted_vis_list = predict_arlexecute(future_vis_graph, gleam_model,
                                           context=jspar["modeling"]["predict"]["context"],  #'wstack'
                                           vis_slices=vis_slices)
    predicted_vis_list = arlexecute.compute(predicted_vis_list, sync=True)
    corrupted_vis_list = corrupt_arlexecute(predicted_vis_list, phase_error=jspar["modeling"]["corrupt"]["phase_error"]) #1.0

    LOG.info('About to run corrupt to get corrupted visibility')
    corrupted_vis_list = arlexecute.compute(corrupted_vis_list, sync=True)

    LOG.info('About to output predicted_vislist.hdf')
    export_blockvisibility_to_hdf5(predicted_vis_list,
                                   '%s/%s' % (RESULTS_DIR,jspar["files"]["predicted_vis_list"]))

    LOG.info('About to output corrupted_vislist.hdf')

    export_blockvisibility_to_hdf5(corrupted_vis_list,
                                   '%s/%s' % (RESULTS_DIR, jspar["files"]["corrupted_vis_list"]))
    # Close Dask client
    arlexecute.close()
465829
def client(self):
        
        if self._client is None:
            self._client = Client_(self.servers)
        return self._client
524031
def get_client(self):
        
        backend_class = import_string(current_app.config.get('CACHE_BACKEND'))
        backend = backend_class(**current_app.config.get('CACHE_BACKEND_OPTIONS'))
        return backend
1370377
def ds_list(self):      # NOTE: how to name to distinguish from locally configured datastream objects?
        
        # FIXME: how to make access to a versioned ds_list ?

        if self._ds_list is None:
            self._ds_list = self._get_datastreams()
        return self._ds_list
1043358
def _read(self):
        
        if self._dask:
            d = da.from_delayed(
                delayed(read_from_bpch, )(
                    self.filename, self.file_position, self.shape,
                    self.dtype, self.endian, use_mmap=self._mmap
                ),
                self.shape, self.dtype
            )
        else:
            d = read_from_bpch(
                    self.filename, self.file_position, self.shape,
                    self.dtype, self.endian, use_mmap=self._mmap
            )

        return d
473029
def _load(self, files_in, files_out, urlpath, meta=True):
        """"""
        import dask
        out = []
        outnames = []
        for file_in, file_out in zip(files_in, files_out):
            cache_path = file_out.path
            outnames.append(cache_path)

            # If `_munge_path` did not find a match we want to avoid
            # writing to the urlpath.
            if cache_path == urlpath:
                continue

            if not os.path.isfile(cache_path):
                logger.debug("Caching file: {}".format(file_in.path))
                logger.debug("Original path: {}".format(urlpath))
                logger.debug("Cached at: {}".format(cache_path))
                if meta:
                    self._log_metadata(urlpath, file_in.path, cache_path)
                ddown = dask.delayed(_download)
                out.append(ddown(file_in, file_out, self.blocksize,
                                 self.output))
        dask.compute(*out)
        return outnames
720199
def _get_client():
    '''
    
    '''
    client = salt.cloud.CloudClient(
            os.path.join(os.path.dirname(__opts__['conf_file']), 'cloud')
            )
    return client