Loads a vector of fits
487772
def _load_vector_fit(self, fit_key, h5file):
        
        vector_fit = []
        for i in range(len(h5file[fit_key].keys())):
            fit_data = self._read_dict(h5file[fit_key]['comp_%d'%i])
            vector_fit.append(self._load_scalar_fit(fit_data=fit_data))
        return vector_fit
617155
def _load_into_numpy(sf, np_array, start, end, strides=None, shape=None):
    """"""
    np_array[:] = 0.0
    np_array_2d = np_array.reshape((np_array.shape[0], np_array.shape[1] * np_array.shape[2]))
    _extensions.sframe_load_to_numpy(sf, np_array.ctypes.data,
                                     np_array_2d.strides, np_array_2d.shape,
                                     start, end)
893604
def savefits(cube, fitsname, **kwargs):
    
    ### pick up kwargs
    dropdeg = kwargs.pop('dropdeg', False)
    ndim    = len(cube.dims)

    ### load yaml
    FITSINFO = get_data('decode', 'data/fitsinfo.yaml')
    hdrdata = yaml.load(FITSINFO, dc.utils.OrderedLoader)

    ### default header
    if ndim == 2:
        header = fits.Header(hdrdata['dcube_2d'])
        data   = cube.values.T
    elif ndim == 3:
        if dropdeg:
            header = fits.Header(hdrdata['dcube_2d'])
            data   = cube.values[:, :, 0].T
        else:
            header = fits.Header(hdrdata['dcube_3d'])

            kidfq     = cube.kidfq.values
            freqrange = ~np.isnan(kidfq)
            orderedfq = np.argsort(kidfq[freqrange])
            newcube   = cube[:, :, orderedfq]
            data      = newcube.values.T
    else:
        raise TypeError(ndim)

    ### update Header
    if cube.coordsys == 'AZEL':
        header.update({'CTYPE1': 'dAZ', 'CTYPE2': 'dEL'})
    elif cube.coordsys == 'RADEC':
        header.update({'OBSRA': float(cube.xref), 'OBSDEC': float(cube.yref)})
    else:
        pass
    header.update({'CRVAL1': float(cube.x[0]),
                   'CDELT1': float(cube.x[1] - cube.x[0]),
                   'CRVAL2': float(cube.y[0]),
                   'CDELT2': float(cube.y[1] - cube.y[0]),
                   'DATE': datetime.now(timezone('UTC')).isoformat()})
    if (ndim == 3) and (not dropdeg):
        header.update({'CRVAL3': float(newcube.kidfq[0]),
                       'CDELT3': float(newcube.kidfq[1] - newcube.kidfq[0])})

    fitsname = str(Path(fitsname).expanduser())
    fits.writeto(fitsname, data, header, **kwargs)
    logger.info('{} has been created.'.format(fitsname))
113401
def load_inhibit(self, train=True, test=True) -> tuple:
        """"""

        def loader(kws: list, nkws: list):
            from precise.params import pr
            inputs = np.empty((0, pr.n_features, pr.feature_size))
            outputs = np.zeros((len(kws), 1))
            for f in kws:
                if not isfile(f):
                    continue
                new_vec = load_vector(f, vectorize_inhibit)
                inputs = np.concatenate([inputs, new_vec])

            return self.merge((inputs, outputs), self.__load_files(kws, nkws))

        return self.__load(loader, train, test)
28632
def _load_scalar_fit(self, fit_key=None, h5file=None, fit_data=None):
        
        if (fit_key is None) ^ (h5file is None):
            raise ValueError("Either specify both fit_key and h5file, or"
                " neither")

        if not ((fit_key is None) ^ (fit_data is None)):
            raise ValueError("Specify exactly one of fit_key and fit_data.")

        if fit_data is None:
            fit_data = self._read_dict(h5file[fit_key])

        if 'fitType' in fit_data.keys() and fit_data['fitType'] == 'GPR':
            fit = _eval_pysur.evaluate_fit.getGPRFitAndErrorEvaluator(fit_data)
        else:
            fit = _eval_pysur.evaluate_fit.getFitEvaluator(fit_data)

        return fit
782008
def _load(pathtovector,
              wordlist,
              num_to_load=None,
              truncate_embeddings=None,
              sep=" "):
        """"""
        vectors = []
        addedwords = set()
        words = []

        try:
            wordlist = set(wordlist)
        except ValueError:
            wordlist = set()

        logger.info("Loading {0}".format(pathtovector))

        firstline = open(pathtovector).readline().strip()
        try:
            num, size = firstline.split(sep)
            num, size = int(num), int(size)
            logger.info("Vector space: {} by {}".format(num, size))
            header = True
        except ValueError:
            size = len(firstline.split(sep)) - 1
            logger.info("Vector space: {} dim, # items unknown".format(size))
            word, rest = firstline.split(sep, 1)
            # If the first line is correctly parseable, set header to False.
            header = False

        if truncate_embeddings is None or truncate_embeddings == 0:
            truncate_embeddings = size

        for idx, line in enumerate(open(pathtovector, encoding='utf-8')):

            if header and idx == 0:
                continue

            word, rest = line.rstrip(" \n").split(sep, 1)

            if wordlist and word not in wordlist:
                continue

            if word in addedwords:
                raise ValueError("Duplicate: {} on line {} was in the "
                                 "vector space twice".format(word, idx))

            if len(rest.split(sep)) != size:
                raise ValueError("Incorrect input at index {}, size "
                                 "is {}, expected "
                                 "{}".format(idx+1,
                                             len(rest.split(sep)), size))

            words.append(word)
            addedwords.add(word)
            vectors.append(np.fromstring(rest, sep=sep)[:truncate_embeddings])

            if num_to_load is not None and len(addedwords) >= num_to_load:
                break

        vectors = np.array(vectors).astype(np.float32)

        logger.info("Loading finished")
        if wordlist:
            diff = wordlist - addedwords
            if diff:
                logger.info("Not all items from your wordlist were in your "
                            "vector space: {}.".format(diff))

        return vectors, words
47670
def _evaluate_fits(self, x, fit_key):
        
        fit = self.fits[fit_key]
        fit_params = self._get_fit_params(np.copy(x), fit_key)
        if type(fit) == list:
            res = []
            for i in range(len(fit)):
                res.append(fit[i](fit_params))
            return np.array(res)
        else:
            return fit(fit_params)
714839
function _Vector(dimension, data) {
  if (!dimension || !Number.isInteger(dimension))
    throw new TypeError('Dimension of the array must be an integer.');

  var _values;
  if (!data || data === undefined)
    _values = [];
  else if (!util.isNumberArray(data))
    throw new TypeError('Invalid values for vector.');
  else
    _values = data;

  if (_values.length < dimension) {
    var zeros = Array(dimension - _values.length).fill(0);
    _values = _values.concat(zeros);
  } else if (_values.length > dimension) {
    _values = _values.slice(0, dimension);
  }

  readonly(this, "dimension", dimension);
  readonly(this, "values", _values);
  readonly(this, "magnitude", _magnitudeOf(_values));
}
1296040
def _load_int(self):
        """"""
        values = numpy.fromfile(self.filepath_int)
        if self.NDIM > 0:
            values = values.reshape(self.seriesshape)
        return values
29523
def returnMatches(kw, kv, n, theta, inputScaling=1.0):
  
  # How many weight vectors and input vectors to generate at a time
  m1 = 4
  m2 = 1000

  weights = getSparseTensor(kw, n, m1, fixedRange=1.0 / kw)

  # Initialize random input vectors using given scaling and see how many match
  inputVectors = getSparseTensor(kv, n, m2,
                                 onlyPositive=True,
                                 fixedRange= 2*inputScaling / kw,
                                 )
  dot = inputVectors.matmul(weights.t())
  numMatches = ((dot >= theta).sum()).item()
  pctMatches = numMatches / float(m1*m2)

  return pctMatches, numMatches, m1*m2
347193
public void fit() {
        val props = Nd4j.getExecutioner().getEnvironmentInformation();
        if (props.getProperty("backend").equals("CUDA")) {
            if (Nd4j.getAffinityManager().getNumberOfDevices() > 1)
                throw new IllegalStateException("Multi-GPU word2vec/doc2vec isn't available atm");
            //if (!NativeOpsHolder.getInstance().getDeviceNativeOps().isP2PAvailable())
            //throw new IllegalStateException("Running Word2Vec on multi-gpu system requires P2P support between GPUs, which looks to be unavailable on your system.");
        }

        Nd4j.getRandom().setSeed(configuration.getSeed());

        AtomicLong timeSpent = new AtomicLong(0);
        if (!trainElementsVectors && !trainSequenceVectors)
            throw new IllegalStateException(
                            "You should define at least one training goal 'trainElementsRepresentation' or 'trainSequenceRepresentation'");
        if (iterator == null)
            throw new IllegalStateException("You can't fit() data without SequenceIterator defined");

        if (resetModel || (lookupTable != null && vocab != null && vocab.numWords() == 0)) {
            // build vocabulary from scratches
            buildVocab();
        }

        WordVectorSerializer.printOutProjectedMemoryUse(vocab.numWords(), configuration.getLayersSize(),
                        configuration.isUseHierarchicSoftmax() && configuration.getNegative() > 0 ? 3 : 2);

        if (vocab == null || lookupTable == null || vocab.numWords() == 0)
            throw new IllegalStateException("You can't fit() model with empty Vocabulary or WeightLookupTable");

        // if model vocab and lookupTable is built externally we basically should check that lookupTable was properly initialized
        if (!resetModel || existingModel != null) {
            lookupTable.resetWeights(false);
        } else {
            // otherwise we reset weights, independent of actual current state of lookup table
            lookupTable.resetWeights(true);

            // if preciseWeights used, we roll over data once again
            if (configuration.isPreciseWeightInit()) {
                log.info("Using precise weights init...");
                iterator.reset();

                while (iterator.hasMoreSequences()) {
                    val sequence = iterator.nextSequence();

                    // initializing elements, only once
                    for (T element : sequence.getElements()) {
                        T realElement = vocab.tokenFor(element.getLabel());

                        if (realElement != null && !realElement.isInit()) {
                            val rng = Nd4j.getRandomFactory().getNewRandomInstance(
                                            configuration.getSeed() * realElement.hashCode(),
                                            configuration.getLayersSize() + 1);

                            val randArray = Nd4j.rand(new int[] {1, configuration.getLayersSize()}, rng).subi(0.5)
                                            .divi(configuration.getLayersSize());

                            lookupTable.getWeights().getRow(realElement.getIndex(), true).assign(randArray);
                            realElement.setInit(true);
                        }
                    }

                    // initializing labels, only once
                    for (T label : sequence.getSequenceLabels()) {
                        T realElement = vocab.tokenFor(label.getLabel());

                        if (realElement != null && !realElement.isInit()) {
                            Random rng = Nd4j.getRandomFactory().getNewRandomInstance(
                                            configuration.getSeed() * realElement.hashCode(),
                                            configuration.getLayersSize() + 1);
                            INDArray randArray = Nd4j.rand(new int[] {1, configuration.getLayersSize()}, rng).subi(0.5)
                                            .divi(configuration.getLayersSize());

                            lookupTable.getWeights().getRow(realElement.getIndex(), true).assign(randArray);
                            realElement.setInit(true);
                        }
                    }
                }

                this.iterator.reset();
            }
        }

        initLearners();
        initIntersectVectors();

        log.info("Starting learning process...");
        timeSpent.set(System.currentTimeMillis());
        if (this.stopWords == null)
            this.stopWords = new ArrayList<>();

        val wordsCounter = new AtomicLong(0);
        for (int currentEpoch = 1; currentEpoch <= numEpochs; currentEpoch++) {
            val linesCounter = new AtomicLong(0);


            val sequencer = new AsyncSequencer(this.iterator, this.stopWords);
            sequencer.start();

            val timer = new AtomicLong(System.currentTimeMillis());
            val thread = new VectorCalculationsThread(0, currentEpoch, wordsCounter, vocab.totalWordOccurrences(),
                    linesCounter, sequencer, timer, numEpochs);
            thread.start();

            try {
                sequencer.join();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

            try {
               thread.join();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

            // TODO: fix this to non-exclusive termination
            if (trainElementsVectors && elementsLearningAlgorithm != null
                            && (!trainSequenceVectors || sequenceLearningAlgorithm == null)
                            && elementsLearningAlgorithm.isEarlyTerminationHit()) {
                break;
            }

            if (trainSequenceVectors && sequenceLearningAlgorithm != null
                            && (!trainElementsVectors || elementsLearningAlgorithm == null)
                            && sequenceLearningAlgorithm.isEarlyTerminationHit()) {
                break;
            }
            log.info("Epoch [" + currentEpoch + "] finished; Elements processed so far: [" + wordsCounter.get()
                            + "];  Sequences processed: [" + linesCounter.get() + "]");

            if (eventListeners != null && !eventListeners.isEmpty()) {
                for (VectorsListener listener : eventListeners) {
                    if (listener.validateEvent(ListenerEvent.EPOCH, currentEpoch))
                        listener.processEvent(ListenerEvent.EPOCH, this, currentEpoch);
                }
            }
        }

        log.info("Time spent on training: {} ms", System.currentTimeMillis() - timeSpent.get());
    }
522928
def load(self, data, size=None):
        """"""
        self.bind()
        if size is None:
            # ffi's sizeof understands arrays
            size = sizeof(data)
        if size == self.buffer_size:
            # same size - no need to allocate new buffer, just copy
            glBufferSubData(
                self.array_type,
                0,
                size,
                to_raw_pointer(data)
            )
        else:
            # buffer size has changed - need to allocate new buffer in the GPU
            glBufferData(
                self.array_type,
                size,
                to_raw_pointer(data),
                self.draw_type
            )
            self.buffer_size = size
        self.unbind()
1363649
def create_from_fits(cls, fitsfile, norm_type='flux'):
        
        tsmap = WcsNDMap.read(fitsfile)

        tab_e = Table.read(fitsfile, 'EBOUNDS')
        tab_s = Table.read(fitsfile, 'SCANDATA')
        tab_f = Table.read(fitsfile, 'FITDATA')

        tab_e = convert_sed_cols(tab_e)
        tab_s = convert_sed_cols(tab_s)
        tab_f = convert_sed_cols(tab_f)

        emin = np.array(tab_e['e_min'])
        emax = np.array(tab_e['e_max'])
        try:
            if str(tab_e['e_min'].unit) == 'keV':
                emin /= 1000.
        except:
            pass
        try:
            if str(tab_e['e_max'].unit) == 'keV':
                emax /= 1000.
        except:
            pass

        nebins = len(tab_e)
        npred = tab_e['ref_npred']

        ndim = len(tsmap.data.shape)

        if ndim == 2:
            cube_shape = (tsmap.data.shape[0],
                          tsmap.data.shape[1], nebins)
        elif ndim == 1:
            cube_shape = (tsmap.data.shape[0], nebins)
        else:
            raise RuntimeError("Counts map has dimension %i" % (ndim))

        refSpec = ReferenceSpec.create_from_table(tab_e)
        nll_vals = -np.array(tab_s["dloglike_scan"])
        norm_vals = np.array(tab_s["norm_scan"])

        axis = MapAxis.from_edges(np.concatenate((emin, emax[-1:])),
                                  interp='log')
        geom_3d = tsmap.geom.to_cube([axis])
        tscube = WcsNDMap(geom_3d,
                          np.rollaxis(tab_s["ts"].reshape(cube_shape), 2, 0))

        ncube = WcsNDMap(geom_3d,
                         np.rollaxis(tab_s["norm"].reshape(cube_shape), 2, 0))
        nmap = WcsNDMap(tsmap.geom,
                        tab_f['fit_norm'].reshape(tsmap.data.shape))

        ref_colname = 'ref_%s' % norm_type
        norm_vals *= tab_e[ref_colname][np.newaxis, :, np.newaxis]

        return cls(tsmap, nmap, tscube, ncube,
                   norm_vals, nll_vals, refSpec,
                   norm_type)
1112539
def load_coef(filename):
    """"""
    
    with open(filename) as f: 
        lines = f.readlines()

        lst = lines[0].split(',')

        nmax = int(lst[0])
        mmax = int(lst[1])

        L = (nmax + 1) + mmax * (2 * nmax - mmax + 1);

        vec = np.zeros(L, dtype=np.complex128)
   
        lines.pop(0)

        for n, line in enumerate(lines):
            lst = line.split(',')
            re = float(lst[0])
            im = float(lst[1])
            vec[n] = re + 1j * im

    return sp.ScalarCoefs(vec, nmax, mmax)
385860
def vectorize(json_file, vec, dofit=True):
    
    ## CountVectorizer, efficiently.
    screen_names = [x[0] for x in extract_tweets(json_file)]
    if dofit:
        X = vec.fit_transform(x[1] for x in extract_tweets(json_file))
    else:
        X = vec.transform(x[1] for x in extract_tweets(json_file))
    return screen_names, X
613453
def fits_list(filter,root,fnames):
    """"""
    import re, pyfits, wcsutil
    for file in fnames:
        if re.match(filter,file):
            fh=pyfits.open(file)
            for ext in fh:
                obj=ext.header.get('OBJECT',file)
                dx=ext.header.get('NAXIS1',None)
                dy=ext.header.get('NAXIS2',None)
                wcs=wcsutil.WCSObject(ext)
                (x1,y1)=wcs.xy2rd((1,1,))
                (x2,y2)=wcs.xy2rd((dx,dy))
                ccds=[x1,y1,x2,y2]
                pointing={'label': obj, 'camera': ccds}
    return files
206066
def manage_fits(list_of_frame):
    """"""

    import astropy.io.fits as fits
    import numina.types.dataframe as df

    refs = []
    for frame in list_of_frame:
        if isinstance(frame, str):
            ref = fits.open(frame)
            refs.append(ref)
        elif isinstance(frame, fits.HDUList):
            refs.append(frame)
        elif isinstance(frame, df.DataFrame):
            ref = frame.open()
            refs.append(ref)
        else:
            refs.append(frame)
    try:
        yield refs
    finally:
        # release
        for obj in refs:
            obj.close()