Writes all the cached NDArrays to disk and empties the cache
489506
def _write_ndarrays_cache_to_disk(self):
        """"""
        for h5loc, arrs in self._ndarrays_cache.items():
            title = arrs[0].title
            chunkshape = (self.chunksize,) + arrs[0].shape[1:] if self.chunksize is not\
                                                           None else None

            arr = NDArray(np.concatenate(arrs), h5loc=h5loc, title=title)

            if h5loc not in self._ndarrays:
                loc, tabname = os.path.split(h5loc)
                ndarr = self.h5file.create_earray(
                    loc,
                    tabname,
                    tb.Atom.from_dtype(arr.dtype),
                    (0, ) + arr.shape[1:],
                    chunkshape=chunkshape,
                    title=title,
                    filters=self.filters,
                    createparents=True,
                )
                self._ndarrays[h5loc] = ndarr
            else:
                ndarr = self._ndarrays[h5loc]

            idx_table_h5loc = h5loc + '_indices'
            if idx_table_h5loc not in self.indices:
                self.indices[idx_table_h5loc] = HDF5IndexTable(idx_table_h5loc)
            idx_tab = self.indices[idx_table_h5loc]

            for arr_length in (len(a) for a in arrs):
                idx_tab.append(arr_length)

            ndarr.append(arr)

        self._ndarrays_cache = defaultdict(list)
544936
def _create_empty_array(self, frames, always_2d, dtype):
        """"""
        import numpy as np
        if always_2d or self.channels > 1:
            shape = frames, self.channels
        else:
            shape = frames,
        return np.empty(shape, dtype, order='C')
653412
def clear(self):
        ""
        if isinstance(self.data, np.ndarray):
            data = self.data[:, :0]
        elif util.pd and isinstance(self.data, util.pd.DataFrame):
            data = self.data.iloc[:0]
        elif isinstance(self.data, dict):
            data = {k: v[:0] for k, v in self.data.items()}
        with util.disable_constant(self):
            self.data = data
        self.send(data)
1278759
public void clear() {
        if (mSize != 0) {
            freeArrays(mHashes, mArray, mSize);
            mHashes = EMPTY_INTS;
            mArray = EMPTY_OBJECTS;
            mSize = 0;
        }
    }
825244
public void batchWrite(List<? extends Object> objectsToWrite, List<? extends Object> objectsToDelete) {
        batchWrite(objectsToWrite, objectsToDelete, this.config);
    }
374935
public static void writeArrayToDisk(INDArray arr, File toWrite) throws IOException {
        try (FileOutputStream os = new FileOutputStream(toWrite)) {
            FileChannel channel = os.getChannel();
            ByteBuffer buffer = BinarySerde.toByteBuffer(arr);
            channel.write(buffer);
        }
    }
498716
def write_ndjson(filename, data_iterable, append=False, **kwargs):
    
    write_mode = "ab" if append else "wb"
    logger.info("writing to file {}".format(filename))
    with codecs.open(filename, write_mode, "utf-8") as outfile:
        for item in data_iterable:
            outfile.write(json.dumps(item) + "\n")
            yield item
1144039
def store(self, t, step):
        
        max_cache = int(self.system.tds.config.max_cache)
        if len(self.vars) >= max_cache > 0:
            self.dump()
            self.vars = list()
            self.t = list()
            self.k = list()
            logger.debug(
                'varout cache cleared at simulation t = {:g}.'.format(
                    self.system.dae.t))
            self._mode = 'a'

        var_data = matrix([self.system.dae.x, self.system.dae.y])

        # ===== This code block is deprecated =====
        self.t.append(t)
        self.k.append(step)
        self.vars.append(var_data)
        # =========================================

        # clear data cache if written to disk
        if self.np_nrows >= max_cache > 0:
            self.dump_np_vars()
            self.np_vars = np.zeros(self._np_block_shape)
            self.np_nrows = 0
            self.np_t = np.zeros((self._np_block_rows,))
            self.np_k = np.zeros((self._np_block_rows,))
            logger.debug(
                'np_vars cache cleared at simulation t = {:g}.'.format(
                    self.system.dae.t))
            self._mode = 'a'

        # initialize before first-time adding data
        if self.np_nrows == 0:
            self.np_ncols = len(var_data)
            self._np_block_shape = (self._np_block_rows, self.np_ncols)
            self.np_vars = np.zeros(self._np_block_shape)
            self.np_t = np.zeros((self._np_block_rows,))
            self.np_k = np.zeros((self._np_block_rows,))

        # adding data to the matrix
        # self.np_vars[self.np_nrows, 0] = t
        self.np_t[self.np_nrows] = t
        self.np_k[self.np_nrows] = step
        self.np_vars[self.np_nrows, :] = np.array(var_data).reshape((-1))
        self.np_nrows += 1

        # check if matrix extension is needed
        if self.np_nrows >= self.np_vars.shape[0]:
            self.np_vars = np.concatenate([self.np_vars, np.zeros(self._np_block_shape)], axis=0)
            self.np_t = np.concatenate([self.np_t, np.zeros((self._np_block_rows,))], axis=0)
            self.np_k = np.concatenate([self.np_k, np.zeros((self._np_block_rows,))], axis=0)

        # remove the post-computed variables from the variable list
        if self.system.tds.config.compute_flows:
            self.system.dae.y = self.system.dae.y[:self.system.dae.m]
569561
def flush(self):
        
        items = []

        for data in self._to_put:
            items.append(encode_put(self.connection.dynamizer, data))

        for data in self._to_delete:
            items.append(encode_delete(self.connection.dynamizer, data))
        self._write(items)
        self._to_put = []
        self._to_delete = []
103601
@Override
    public void clear() {
        Map<String, GroovyRunner> map = getMap();
        writeLock.lock();
        try {
            cachedValues = null;
            map.clear();
            loadDefaultRunners();
        } finally {
            writeLock.unlock();
        }
    }
493547
def Flush(self):
    """"""
    if self._dirty:
      self.Set(self.Schema.SIZE(self.size))
      if self.content_last is not None:
        self.Set(self.Schema.CONTENT_LAST, self.content_last)

    # Flushing the cache will write all chunks to the blob store.
    self.chunk_cache.Flush()
    super(AFF4ImageBase, self).Flush()
1220268
def _write_to_cache(self, expr, res):
        """"""
        res = dedent(res)
        super()._write_to_cache(expr, res)
830818
def clear(self):
        """"""
        if hasattr(self, '_bytes_012'):
            self._bytes_012.fill(0)
            self._bytes_345.fill(0)
        self._text_lines = [] * self._n_rows
        self._pending_writes = []
512631
@Override
    public void collect(INDArray... arrays) {
        // we basically want to free memory, without touching INDArray itself.
        // so we don't care when gc is going to release object: memory is already cached

        Nd4j.getExecutioner().commit();

        int cnt = -1;
        AtomicAllocator allocator = AtomicAllocator.getInstance();
        for (INDArray array : arrays) {
            cnt++;
            // we don't collect views, since they don't have their own memory
            if (array == null || array.isView())
                continue;

            AllocationPoint point = allocator.getAllocationPoint(array);

            if (point.getAllocationStatus() == AllocationStatus.HOST)
                allocator.getMemoryHandler().free(point, AllocationStatus.HOST);
            else if (point.getAllocationStatus() == AllocationStatus.DEVICE) {
                allocator.getMemoryHandler().free(point, AllocationStatus.DEVICE);
                allocator.getMemoryHandler().free(point, AllocationStatus.HOST);
            } else if (point.getAllocationStatus() == AllocationStatus.DEALLOCATED) {
                // do nothing
            } else
                throw new RuntimeException(
                                "Unknown AllocationStatus: " + point.getAllocationStatus() + " for argument: " + cnt);

            point.setAllocationStatus(AllocationStatus.DEALLOCATED);
        }
    }
66993
def reset(self):
        """"""
        self.data     = []
        self.size     = 0
        self.kdtree   = None  # KDTree
        self.nn_ready = False
20105
def distribute(self, cache):
        
        data = self._features_as_kv()
        for key in data.keys():
            cache.batch_set(data[key])
1282768
def saveAsTFRecords(df, output_dir):
  
  tf_rdd = df.rdd.mapPartitions(toTFExample(df.dtypes))
  tf_rdd.saveAsNewAPIHadoopFile(output_dir, "org.tensorflow.hadoop.io.TFRecordFileOutputFormat",
                                keyClass="org.apache.hadoop.io.BytesWritable",
                                valueClass="org.apache.hadoop.io.NullWritable")