Perform update of param_arrays from grad_arrays on NCCL kvstore.
974210
def _update_params_on_kvstore_nccl(param_arrays, grad_arrays, kvstore, param_names):
    """"""
    valid_indices = [index for index, grad_list in
                     enumerate(grad_arrays) if grad_list[0] is not None]
    valid_grad_arrays = [grad_arrays[i] for i in valid_indices]
    valid_param_arrays = [param_arrays[i] for i in valid_indices]
    valid_param_names = [param_names[i] for i in valid_indices]
    size = len(valid_grad_arrays)
    start = 0
    # Use aggregation by default only with NCCL
    default_batch = '16'
    batch = int(os.getenv('MXNET_UPDATE_AGGREGATION_SIZE', default_batch))
    while start < size:
        end = start + batch if start + batch < size else size
        # push gradient, priority is negative index
        kvstore.push(valid_param_names[start:end], valid_grad_arrays[start:end], priority=-start)
        # pull back the weights
        kvstore.pull(valid_param_names[start:end], valid_param_arrays[start:end], priority=-start)
        start = end
1079641
def load_from_array(self, keys, data_array):
        

        if len(keys) != np.shape(data_array)[1]:
            raise ValueError('Key list does not match shape of array!')

        for i, key in enumerate(keys):
            if key in self.INT_ATTRIBUTE_LIST:
                self.data[key] = data_array[:, i].astype(int)
            else:
                self.data[key] = data_array[:, i]
            if key not in self.TOTAL_ATTRIBUTE_LIST:
                print('Key %s not a recognised catalogue attribute' % key)

        self.update_end_year()
1126369
def updateParams(self, newvalues, update_all=False):
        """"""
        assert all(map(lambda x: x in self.freeparams, newvalues.keys())),\
                "Invalid entry in newvalues: {0}\nfreeparams: {1}".format(
                ', '.join(newvalues.keys()), ', '.join(self.freeparams))

        newvalues_list = [{} for k in range(self.ncats)]

        if update_all or any([param in self.distributionparams for param
                in newvalues.keys()]):
            self._d_distributionparams = {}
            for param in self.distributionparams:
                if param in newvalues:
                    _checkParam(param, newvalues[param], self.PARAMLIMITS,
                            self.PARAMTYPES)
                    setattr(self, param, copy.copy(newvalues[param]))
            self._lambdas = DiscreteGamma(self.alpha_lambda, self.beta_lambda,
                    self.ncats)
            for (k, l) in enumerate(self._lambdas):
                newvalues_list[k][self.distributedparam] = l
        for name in self.freeparams:
            if name not in self.distributionparams:
                if name in newvalues:
                    value = newvalues[name]
                    _checkParam(name, value, self.PARAMLIMITS, self.PARAMTYPES)
                    setattr(self, name, copy.copy(value))
                    for k in range(self.ncats):
                        newvalues_list[k][name] = value
                elif update_all:
                    for k in range(self.ncats):
                        newvalues_list[k][name] = getattr(self, name)

        assert len(newvalues_list) == len(self._models) == self.ncats
        for (k, newvalues_k) in enumerate(newvalues_list):
            self._models[k].updateParams(newvalues_k)

        # check to make sure all models have same parameter values
        for param in self.freeparams:
            if param not in self.distributionparams:
                pvalue = getattr(self, param)
                assert all([scipy.allclose(pvalue, getattr(model, param))
                        for model in self._models]), ("{0}\n{1}".format(
                        pvalue, '\n'.join([str(getattr(model, param))
                        for model in self._models])))
39005
@Override
    public void applyUpdate(StepFunction function, INDArray params, INDArray grad, boolean isFinalStep) {

        try {
            updatesLock.readLock().lock();

            firstOne.compareAndSet(-1L, Thread.currentThread().getId());

            if (hasSomething.get())
                function.step(params, updates);

            barrier.await();
            if (firstOne.get() == Thread.currentThread().getId()) {
                // one thread just nullifies this array
                updates.assign(0.0);
                hasSomething.set(false);

                firstOne.set(-1L);
            }

            updatesLock.readLock().unlock();
            barrier.await();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new RuntimeException(e);
        } catch (BrokenBarrierException e) {
            throw new RuntimeException(e);
        }
    }
983822
def update(self, attrs):
        """"""
        ########################################################################
        def convertStrToDict(strVal):
            if isinstance(strVal, dict): return strVal
            strVal = re.sub("[\{\}]+", "", str(strVal))
            regexCol = re.compile(":")
            terms = re.split("[,\s]+", strVal)
            keyvals = [re.split(regexCol, t) for t in terms]
            x = re.compile("['\"]")
            ret = {}
            boolTrue  = re.compile("true" , flags=re.IGNORECASE)
            boolFalse = re.compile("false", flags=re.IGNORECASE)
            for k, v in keyvals:
                k = re.sub(x, "", k)
                v = re.sub(x, "", v)
                if   re.search(boolTrue, v):    v = True
                elif re.search(boolFalse, v):   v = False
                else:
                    if '.' in v:
                        try:                    v = float(v)
                        except:                 pass
                    else:
                        try:                    v = int(v)
                        except:                 pass
                ret[k] = v
            return ret
        ########################################################################
        self._validateAttrs(attrs)
        for k,v in iteritems(attrs):
            typecast = type( getattr(self, k) )
            if typecast==bool and v=="False":   newval = False # "False" evalued as boolean is True because its length > 0
            elif issubclass(typecast, c.RestrictedType): # let the RestrictedType handle the type setting, value matching
                                                newval = typecast(v)
            elif "<" in str(v) or v==None:      newval = typecast(v)
            elif k == "initCmd":                newval = str(v) # specifically don't mangle the command as specified
            elif k == "initOptions":            newval = convertStrToDict(v)
            else:                               newval = typecast(str(v).lower())
            setattr(self, k, newval)
        if self.isComputer: pass
        elif "difficulty" in attrs and attrs["difficulty"]!=None: # the final state of this PlayerRecord cannot be a non-computer and specify a difficulty
            raise ValueError("%s type %s=%s does not have a difficulty"%(
                self.__class__.__name__, self.type.__class__.__name__, self.type.type))
        else: self.difficulty = c.ComputerDifficulties(None)
1373513
static String updateQueryAndParametersIndexes(String parsedQuery,
                                                  Map<String, List<Integer>> parameterNamesToIndexes,
                                                  Map<String, Query.ParameterSetter> parameters,
                                                  boolean allowArrayParameters) {
        List<ArrayParameter> arrayParametersSortedAsc = arrayParametersSortedAsc(parameterNamesToIndexes, parameters, allowArrayParameters);
        if(arrayParametersSortedAsc.isEmpty()) {
            return parsedQuery;
        }

        updateParameterNamesToIndexes(parameterNamesToIndexes, arrayParametersSortedAsc);

        return updateQueryWithArrayParameters(parsedQuery, arrayParametersSortedAsc);
    }
296672
def paramsarray(self, value):
        """"""
        nparams = len(self._index_to_param)
        assert (isinstance(value, scipy.ndarray) and value.ndim == 1), (
                "paramsarray must be 1-dim ndarray")
        assert len(value) == nparams, ("Assigning paramsarray to ndarray "
                "of the wrong length.")
        if (self._paramsarray is not None) and all(value == self._paramsarray):
            return # do not need to do anything if value has not changed
        # build `newvalues` to pass to `updateParams`
        newvalues = {}
        vectorized_params = {}
        for (i, param) in self._index_to_param.items():
            if isinstance(param, str):
                newvalues[param] = float(value[i])
            elif isinstance(param, tuple):
                (iparam, iparamindex) = param
                if iparam in vectorized_params:
                    assert iparamindex not in vectorized_params[iparam]
                    vectorized_params[iparam][iparamindex] = float(value[i])
                else:
                    vectorized_params[iparam] = {iparamindex:float(value[i])}
            else:
                raise ValueError("Invalid param type")
        for (param, paramd) in vectorized_params.items():
            assert set(paramd.keys()) == set(range(len(paramd)))
            newvalues[param] = scipy.array([paramd[i] for i in range(len(paramd))],
                    dtype='float')
        self.updateParams(newvalues)
        self._paramsarray = self.paramsarray
1106700
public void updateParameterCount(final EquPart[] equParts, final int myLocInArray)
    {
        setParameterCount(0);

        for (int p = myLocInArray + 1; p < equParts.length; p++)
        {
            final EquPart part = equParts[p];

            if (part.getLevel() <= getLevel())
                break;

            if ((part.getLevel() == (getLevel() + 1)) && part instanceof OpComma)
                setParameterCount(getParameterCount() + 1);
        }

        if (getParameterCount() > 0)
            setParameterCount(getParameterCount() + 1);
    }
425535
def set_grads(params, params_with_grad):
        
        for param, param_w_grad in zip(params, params_with_grad):
            if param.grad is None:
                param.grad = torch.nn.Parameter(torch.empty_like(param))
            param.grad.data.copy_(param_w_grad.grad.data)
131534
def _initialize_kvstore(kvstore, param_arrays, arg_params, param_names, update_on_kvstore):
    """"""
    for idx, param_on_devs in enumerate(param_arrays):
        name = param_names[idx]
        kvstore.init(name, arg_params[name])

        if update_on_kvstore:
            kvstore.pull(name, param_on_devs, priority=-idx)
586819
def update_dataset(self,dataStr,flatten=False):
        '''
        
        '''

        #Load keys and dimensions
        #########################
        dataDim = dataStr.pop('_dimensions',{})
        attrStr = dataStr.pop('_attributes',{})
        ndims = dataDim.pop('_ndims',0)
        dimensions = [dataDim.keys(),dataDim.values()]
        
        keys = dataStr.keys()
        if len(keys) == 0:
            self.warning(2, 'No data loaded')
            return
        self.message(2, 'Loaded variables : '+str(keys))
        
        #Check what is the current variable type
        isStructure = True if isinstance(dataStr[keys[0]],dict) else False
        
#        datalen = [np.size(dataStr[key]) for key in keys]
        datalen = [list(np.shape(dataStr[key]['data'])[::-1]) for key in keys] if isStructure else [list(np.shape(dataStr[key])[::-1]) for key in keys] #Shape is inverted wrt to order of dimensions to be consistent with check_variable
        if isStructure :
            varDim = [list(dataStr[key]['_dimensions'])[1:] for key in keys]
            ind = [where_list(vDim,dimensions[0]) for vDim in varDim] #Dimensions indices from actual variables' dimensions
            
            #Check dimension lengths
#             dimOk = np.array([enum[1][0] == dimensions[1][ind[enum[0]][0]] for enum in enumerate(datalen)])
            dimOk = [any([enum[1][ii] == dimensions[1][jj] for ii,jj in enumerate(ind[enum[0]])]) for enum in enumerate(datalen)]
            

            if any([not d for d in dimOk]) : 
                notOk = np.where(~np.array(dimOk))[0]
                print datalen
                
                self.Error('Problem with {0} variables : {1}'.format(len(notOk),','.join(np.array(dataStr.keys())[notOk])))
        else :
            ind = [where_list(dlen,dimensions[1]) for dlen in datalen] #Dimensions indices from variable length
            if (np.array(ind).sum() == -1)!= 0 : self.Error('At least one variable have not been properly defined')
            
        dimname = [np.array(dimensions[0])[i].tolist() for i in ind]  #Get correspondance between data structure dimensions and variables
                
        curDim, nself=self.get_currentDim()
        createDim=np.array([np.array([w == -1 for w in where_list(j, curDim[0])]) for i,j in enumerate(dimname) ])
        createDim=np.squeeze(createDim)
#        curInd = atools.where_list(dimname_reduced,curDim[0]) #Get correspondance between data structure dimensions and object dimensions
        
#        createDim = (np.array(curInd) == -1) #Get dimensions to be created   
        toCreate = np.array([not self.__dict__.has_key(key) for key in keys])
        
        updateDim=[]  
        
        self.message(2, 'Updating object with '+str(['{0}({1}:{2})'.format(i[0],i[1],i[2]) for i in zip(*(keys,dimname,datalen))]))
        
        #Update variables available in files
        for enum in enumerate(keys) :
            
            ind=enum[0]
            key=enum[1]
            
            #Load variable
            ##############
#            var=dataStr.get(key)
            dum=dataStr.get(key).pop('data') if isStructure else copy.deepcopy(dataStr.get(key)) 
            if flatten :
                if isinstance(dum,dict) :dum['data']=dum['data'].flatten()
                else : dum=dum.flatten()
            
            if not isStructure :
                dum={'_dimensions':dum._dimensions if hasattr(dum,'_dimensions') else {},
                      '_attributes':dum._attributes if hasattr(dum,'_attributes') else {},
                      'data':dum}
            else :
                dumStr=dataStr.get(key)
                dumStr.update({'data':dum})
                dum=dumStr

            dumDim=dimStr(dimname[ind],datalen[ind])

#             if dataStr[key].has_key('_attributes'):
#                 dum.update(dataStr[key]['_attributes'])
#            if isinstance(dum,np.ma.masked_array) : 
            
#            #Get associated dimensions
#            ##################################
#            datalen = datalen[ind]#[len(dataStr[key]) for key in keys]
#            ind = atools.where_list([datalen],dimensions[1])[0]
#            if (ind == -1) : self.Error('Dimensions of current variable ('+key+') have not been properly defined')
#            dimname = dimensions[
            
            #Initialize variable if required
#            if toCreate :
#            updateDim.append(self.create_Variable(key, dum, dimensions={dimname[ind]:datalen[ind]},toCreate=toCreate[ind],createDim=createDim[ind]))
            updateDim.append(self.create_Variable(key, dum, dimensions=dumDim,toCreate=toCreate[ind],createDim=createDim[ind]))

        
        #Extend missing variables
#         missing__keys = list(set(self.par_list).difference(keys))
#         for enum in enumerate(missing__keys) :
#             ind=enum[0]
#             key=enum[1]
#             updateDim.append(self.create_Variable(key, np.ma.repeat(self.dist_to_coast_leuliette.fill_value), dimensions=dumDim,toCreate=False,createDim=False))
        
        
        #Final sequence
        
        zipped_upd=zip(*(np.hstack(dimname)[~np.hstack(createDim)],np.hstack(datalen)[~np.hstack(createDim)]))
        updateDim_List = np.array(list(set(tuple(i) for i in np.array(zipped_upd,dtype='|S16').tolist()))) #2D unique
#        updateDim_List = np.unique(np.array(zipped_upd,dtype='|S16')) #[str(i) for i in datalen]
#        if updateDim_List.size > 0 : updateDim_List.resize((2,updateDim_List.size/2))
#        updateDim_List = np.unique(zip(*(np.array(dimname)[~createDim],np.array(datalen)[~createDim]))) #[str(i) for i in datalen]

        zipped_dims=zip(*(np.hstack(dimname)[np.hstack(createDim)],np.hstack(datalen)[np.hstack(createDim)]))
        createDim_list = np.array(list(set(tuple(i) for i in np.array(zipped_dims,dtype='|S16').tolist()))) #2D unique
#        clist, inv = np.unique(np.array(zipped_dims,dtype='|S16'),return_inverse=True) #RQ : THIS WILL FAIL IF NUMBERS HAVE MORE THAN 16 DIGITS #[str(i) for i in datalen]
#        if createDim_list.size > 0 : createDim_list.resize((2,createDim_list.size/2))
        
#        createDim_list = np.unique(zip(*(np.array(dimname)[createDim],np.array(datalen)[createDim]))) #[str(i) for i in datalen]
        
        for dname,dim in createDim_list :
            self.create_Dim(dname, np.int(dim))
        
        for dname,dim in updateDim_List:
            self.update_Dim(dname, np.int(dim))
192642
private ArrayDBIDs updateKNNsAndRkNNs(DBIDs ids) {
    ArrayModifiableDBIDs rkNN_ids = DBIDUtil.newArray();
    DBIDs oldids = DBIDUtil.difference(relation.getDBIDs(), ids);
    for(DBIDIter id = oldids.iter(); id.valid(); id.advance()) {
      KNNList oldkNNs = storage.get(id);
      double knnDist = oldkNNs.getKNNDistance();
      // look for new kNNs
      KNNHeap heap = null;
      for(DBIDIter newid = ids.iter(); newid.valid(); newid.advance()) {
        double dist = distanceQuery.distance(id, newid);
        if(dist <= knnDist) {
          // New id changes the kNNs of oldid.
          heap = heap != null ? heap : DBIDUtil.newHeap(oldkNNs);
          heap.insert(dist, newid);
        }
      }
      // kNNs for oldid have changed:
      if(heap != null) {
        KNNList newkNNs = heap.toKNNList();
        storage.put(id, newkNNs);

        // get the difference
        ModifiableDoubleDBIDList added = DBIDUtil.newDistanceDBIDList(),
            removed = DBIDUtil.newDistanceDBIDList();
        DoubleDBIDListIter olditer = oldkNNs.iter(), newiter = newkNNs.iter();
        while(olditer.valid() && newiter.valid()) {
          if(DBIDUtil.equal(olditer, newiter)) {
            olditer.advance();
            newiter.advance();
            continue;
          }
          double newd = newiter.doubleValue(), oldd = olditer.doubleValue();
          if(newd < oldd || (newd == oldd && !oldkNNs.contains(newiter))) {
            added.add(newiter.doubleValue(), newiter);
            newiter.advance();
          }
          else if(oldd < newd || (oldd == newd && !newkNNs.contains(olditer))) {
            removed.add(olditer.doubleValue(), olditer);
            olditer.advance();
          }
          else {
            throw new IllegalStateException("Unexpected third case, needs debug!");
          }
        }
        for(; olditer.valid(); olditer.advance()) {
          removed.add(olditer.doubleValue(), olditer);
        }
        for(; newiter.valid(); newiter.advance()) {
          added.add(newiter.doubleValue(), newiter);
        }
        // add new RkNN
        for(DoubleDBIDListIter newnn = added.iter(); newnn.valid(); newnn.advance()) {
          materialized_RkNN.get(newnn).add(DBIDUtil.newPair(newnn.doubleValue(), id));
        }
        // remove old RkNN
        for(DoubleDBIDListIter oldnn = removed.iter(); oldnn.valid(); oldnn.advance()) {
          materialized_RkNN.get(oldnn).remove(DBIDUtil.newPair(oldnn.doubleValue(), id));
        }

        rkNN_ids.add(id);
      }
    }
    return rkNN_ids;
  }
768736
def process_update(self, update):
        """"""
        data = json.loads(update)
        NetworkTables.getEntry(data["k"]).setValue(data["v"])
848997
def update_params(self, param_d):
        
        for param in self.paramlst:
            if param.name in param_d:
                value = param_d[param.name]
                setattr(self.obj, param.name, value)
310638
def prepare_sparse_params(self, param_rowids):
        '''
        '''
        if not self._kvstore:
            return
        assert(isinstance(param_rowids, dict))
        for param_name, rowids in param_rowids.items():
            if isinstance(rowids, (tuple, list)):
                rowids_1d = []
                for r in rowids:
                    rowids_1d.append(r.reshape((-1,)).astype(np.int64))
                rowid = mx.nd.concat(*rowids_1d, dim=0)
            else:
                rowid = rowids
            param_idx = self._exec_group.param_names.index(param_name)
            param_val = self._exec_group.param_arrays[param_idx]
            self._kvstore.row_sparse_pull(param_name, param_val, row_ids=rowid,
                                          priority=-param_idx)
1312447
private void growDataArrays() {
    currItemsAlloc_ = SamplingUtil.getAdjustedSize(k_, currItemsAlloc_ << rf_.lg());
    if (currItemsAlloc_ == k_) {
      ++currItemsAlloc_;
    }

    data_.ensureCapacity(currItemsAlloc_);
    weights_.ensureCapacity(currItemsAlloc_);
    if (marks_ != null) {
      marks_.ensureCapacity(currItemsAlloc_);
    }
  }
1353318
def _update(self, data):
        """"""
        for k, v in six.iteritems(data):
            new_value = v
            if isinstance(v, dict):
                new_value = type(self)(v)
            elif isinstance(v, list):
                new_value = [(type(self)(e) if isinstance(e, dict) else e)
                             for e in v]
            setattr(self, k, new_value)