Run System.gc() on every node in the H2O cluster.

Having to call this manually from user code is a sign that something is wrong and a better
heuristic is needed internally.
433909
public static void gc() {
    class GCTask extends DTask<GCTask> {
      public GCTask() {super(GUI_PRIORITY);}
      @Override public void compute2() {
        Log.info("Calling System.gc() now...");
        System.gc();
        Log.info("System.gc() finished");
        tryComplete();
      }
    }

    for (H2ONode node : H2O.CLOUD._memary) {
      GCTask t = new GCTask();
      new RPC<>(node, t).call().get();
    }
  }
616044
private   final void dcompute() {// Work to do the distribution
    // Split out the keys into disjointly-homed sets of keys.
    // Find the split point.  First find the range of home-indices.
    H2O cloud = H2O.CLOUD;
    int lo=cloud._memary.length, hi=-1;
    for( Key k : _keys ) {
      int i = k.home(cloud);
      if( i<lo ) lo=i;
      if( i>hi ) hi=i;        // lo <= home(keys) <= hi
    }

    // Classic fork/join, but on CPUs.
    // Split into 3 arrays of keys: lo keys, hi keys and self keys
    final ArrayList<Key> locals = new ArrayList<Key>();
    final ArrayList<Key> lokeys = new ArrayList<Key>();
    final ArrayList<Key> hikeys = new ArrayList<Key>();
    int self_idx = cloud.nidx(H2O.SELF);
    int mid = (lo+hi)>>>1;    // Mid-point
    for( Key k : _keys ) {
      int idx = k.home(cloud);
      if( idx == self_idx ) locals.add(k);
      else if( idx < mid )  lokeys.add(k);
      else                  hikeys.add(k);
    }

    // Launch off 2 tasks for the other sets of keys, and get a place-holder
    // for results to block on.
    _lo = remote_compute(lokeys);
    _hi = remote_compute(hikeys);

    // Setup for local recursion: just use the local keys.
    if( locals.size() != 0 ) {  // Shortcut for no local work
      _local = clone();         // 'this' is completer for '_local', so awaits _local completion
      _local._is_local = true;
      _local._keys = locals.toArray(new Key[locals.size()]); // Keys, including local keys (if any)
      _local.init();            // One-time top-level init
      H2O.submitTask(_local);   // Begin normal execution on a FJ thread
    } else {
      tryComplete();            // No local work, so just immediate tryComplete
    }
  }
1188170
private static synchronized void trackAllocatedHashinatorBytes(long bytes) {
        final long allocated = m_allocatedHashinatorBytes.addAndGet(bytes);
        if (allocated > HASHINATOR_GC_THRESHHOLD) {
            hostLogger.warn(allocated + " bytes of hashinator data has been allocated");
            if (m_emergencyGCThread == null || m_emergencyGCThread.getState() == State.TERMINATED) {
                m_emergencyGCThread = new Thread(new Runnable() {
                    @Override
                    public void run() {
                        hostLogger.warn("Invoking System.gc() to recoup hashinator bytes");
                        System.gc();
                        try {
                            Thread.sleep(2000);
                        } catch (InterruptedException e) {}
                        hostLogger.info(m_allocatedHashinatorBytes.get() + " bytes of hashinator allocated after GC");
                    }
                }, "Hashinator GC thread");
                m_emergencyGCThread.start();
            }
        }
    }
1240351
public static void main(String[] argv) throws Exception {
    org.apache.hadoop.hdfs.DnsMonitorSecurityManager.setTheManager();
    StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG);
    Configuration tconf = new Configuration();
    try {
      argv = DFSUtil.setGenericConf(argv, tconf);
    } catch (IllegalArgumentException e) {
      System.err.println(e.getMessage());
      printUsage("");
      return;
    }
    if (argv.length >= 1) {
      SecondaryNameNode secondary = new SecondaryNameNode(tconf);
      int ret = secondary.processArgs(argv);
      System.exit(ret);
    }

    // Create a never ending deamon
    Daemon checkpointThread = new Daemon(new SecondaryNameNode(tconf)); 
    checkpointThread.start();
  }
516555
void lowerActiveGetCount( H2ONode h2o ) {
    assert _key.home();    // Only the HOME node for a key tracks replicas
    assert h2o != H2O.SELF;// Do not track self as a replica
    while( true ) {        // Repeat, in case racing GETs are bumping the counter
      int old = _rwlock.get(); // Read the lock-word
      assert old > 0;      // Since lowering, must be at least 1
      assert old != -1;    // Not write-locked, because we are an active reader
      assert _replicas.contains(h2o._unique_idx); // Self-bit is set
      if( RW_CAS(old,old-1,"rlock-") ) {
        if( old-1 == 0 )   // GET count fell to zero?
          synchronized( this ) { notifyAll(); } // Notify any pending blocked PUTs
        return;            // Repeat until count is lowered
      }
    }
  }
347095
function (callback) {
            loadArrayBuffer(path.join(dic_path, "cc.dat.gz"), function (err, buffer) {
                if(err) {
                    return callback(err);
                }
                var cc_buffer = new Int16Array(buffer);
                dic.loadConnectionCosts(cc_buffer);
                callback(null);
            });
        }
1260101
private void scheduleLoadTicks() {
        Runnable sendLoadToZookeeper = new Runnable() {
            @Override
            public void run() {
                final List<String> loads = new ArrayList<String>();
                synchronized (meters) {
                    for (Map.Entry<String,Meter> entry : meters.entrySet()) {
                        final String workUnit = entry.getKey();
                        loads.add(String.format("/%s/meta/workload/%s", cluster.name, workUnit));
                        loads.add(String.valueOf(entry.getValue().getOneMinuteRate()));
                    }
                }

                Iterator<String> it = loads.iterator();
                while (it.hasNext()) {
                    final String path = it.next();
                    final String rate = it.next();
                    try {
                        ZKUtils.setOrCreate(cluster.zk, path, rate, CreateMode.PERSISTENT);
                    } catch (Exception e) {
                        // should we fail the loop too?
                        LOG.error("Problems trying to store load rate for {} (value {}): ({}) {}",
                                path, rate, e.getClass().getName(), e.getMessage());
                    }
                }

                String nodeLoadPath = String.format("/%s/nodes/%s", cluster.name, cluster.myNodeID);
                try {
                    NodeInfo myInfo = new NodeInfo(cluster.getState().toString(), cluster.zk.get().getSessionId());
                    byte[] myInfoEncoded = JsonUtil.asJSONBytes(myInfo);
                    ZKUtils.setOrCreate(cluster.zk, nodeLoadPath, myInfoEncoded, CreateMode.EPHEMERAL);
                    if (LOG.isDebugEnabled()) {
                        // TODO: Shouldn't be evaluating methods inside of a logging statement.
                        LOG.debug("My load: {}", myLoad());
                    }
                } catch (Exception e) {
                    LOG.error("Error reporting load info to ZooKeeper.", e);
                }
            }
        };

        loadFuture = cluster.scheduleAtFixedRate(sendLoadToZookeeper, 0, 1, TimeUnit.MINUTES);
    }
469053
public static void set_goals( String msg, boolean oom , long bytes) {
    // Our best guess of free memory, as of the last GC cycle
    final long heapUsed = Boot.HEAP_USED_AT_LAST_GC;
    final long timeGC = Boot.TIME_AT_LAST_GC;
    final long freeHeap = MEM_MAX - heapUsed;
    assert freeHeap >= 0 : "I am really confused about the heap usage; MEM_MAX="+MEM_MAX+" heapUsed="+heapUsed;
    // Current memory held in the K/V store.
    final long cacheUsage = myHisto.histo(false)._cached;
    // Our best guess of POJO object usage: Heap_used minus cache used
    final long pojoUsedGC = Math.max(heapUsed - cacheUsage,0);

    // Block allocations if:
    // the cache is > 7/8 MEM_MAX, OR
    // we cannot allocate an equal amount of POJOs, pojoUsedGC > freeHeap.
    // Decay POJOS_USED by 1/8th every 5 sec: assume we got hit with a single
    // large allocation which is not repeating - so we do not need to have
    // double the POJO amount.
    // Keep at least 1/8th heap for caching.
    // Emergency-clean the cache down to the blocking level.
    long d = MEM_CRITICAL;
    // Decay POJO amount
    long p = pojoUsedGC;
    long age = (System.currentTimeMillis() - timeGC); // Age since last FullGC
    age = Math.min(age,10*60*1000 ); // Clip at 10mins
    while( (age-=5000) > 0 ) p = p-(p>>3); // Decay effective POJO by 1/8th every 5sec
    d -= 2*p - bytes; // Allow for the effective POJO, and again to throttle GC rate
    d = Math.max(d,MEM_MAX>>3); // Keep at least 1/8th heap
    H2O.Cleaner.DESIRED = d;

    String m="";
    if( cacheUsage > H2O.Cleaner.DESIRED ) {
      m = (CAN_ALLOC?"Blocking!  ":"blocked:   ");
      if( oom ) setMemLow(); // Stop allocations; trigger emergency clean
      Boot.kick_store_cleaner();
    } else { // Else we are not *emergency* cleaning, but may be lazily cleaning.
      setMemGood();             // Cache is as low as we'll go; unblock
      if( oom ) {               // But still have an OOM?
        m = "Unblock allocations; cache emptied but memory is low: ";
        // Means the heap is full of uncached POJO's - which cannot be spilled.
        // Here we enter the zone of possibly dieing for OOM.  There's no point
        // in blocking allocations, as no more memory can be freed by more
        // cache-flushing.  Might as well proceed on a "best effort" basis.
        Log.warn(Sys.CLEAN,m+" OOM but cache is emptied:  MEM_MAX = " + PrettyPrint.bytes(MEM_MAX) + ", DESIRED_CACHE = " + PrettyPrint.bytes(d) +", CACHE = " + PrettyPrint.bytes(cacheUsage) + ", POJO = " + PrettyPrint.bytes(p) + ", this request bytes = " + PrettyPrint.bytes(bytes));
      } else { 
        m = "MemGood:   ";
      }
    }

    // No logging if under memory pressure: can deadlock the cleaner thread
    if( Log.flag(Sys.CLEAN) ) {
      String s = m+msg+", HEAP_LAST_GC="+(heapUsed>>20)+"M, KV="+(cacheUsage>>20)+"M, POJO="+(pojoUsedGC>>20)+"M, free="+(freeHeap>>20)+"M, MAX="+(MEM_MAX>>20)+"M, DESIRED="+(H2O.Cleaner.DESIRED>>20)+"M"+(oom?" OOM!":" NO-OOM");
      if( CAN_ALLOC ) Log.debug(Sys.CLEAN ,s);
      else            Log.unwrap(System.err,s);
    }
  }
717692
protected void checkCacheVersion() {
        // Be very aggressive in invalidating the cache.
        boolean invalidate = true;
        QueryRunner run = new QueryRunner(ds);
        try {
            long latestVersion = run.query("SELECT version FROM gw_dataversion", Handlers.LONG_HANDLER); //$NON-NLS-1$
            if (latestVersion > -1 && dataVersion > -1 && latestVersion == dataVersion) {
                invalidate = false;
            } else {
                dataVersion = latestVersion;
            }
        } catch (SQLException e) {
            // TODO need to use the gateway logger to log this!
            e.printStackTrace();
        }
        if (invalidate) {
            invalidateCache();
        }
    }
1305742
def _start_nodes_parallel(self, nodes, max_thread_pool_size):
        
        # Create one thread for each node to start
        thread_pool_size = min(len(nodes), max_thread_pool_size)
        thread_pool = Pool(processes=thread_pool_size)
        log.debug("Created pool of %d threads", thread_pool_size)

        # pressing Ctrl+C flips this flag, which in turn stops the main loop
        # down below
        keep_running = True

        def sigint_handler(signal, frame):
            """
            Makes sure the cluster is saved, before the sigint results in
            exiting during node startup.
            """
            log.error(
                "Interrupted: will save cluster state and exit"
                " after all nodes have started.")
            keep_running = False

        # intercept Ctrl+C
        with sighandler(signal.SIGINT, sigint_handler):
            result = thread_pool.map_async(self._start_node, nodes)
            while not result.ready():
                result.wait(1)
                # check if Ctrl+C was pressed
                if not keep_running:
                    log.error("Aborting upon user interruption ...")
                    # FIXME: `.close()` will keep the pool running until all
                    # nodes have been started; should we use `.terminate()`
                    # instead to interrupt node creation as soon as possible?
                    thread_pool.close()
                    thread_pool.join()
                    self.repository.save_or_update(self)
                    # FIXME: should raise an exception instead!
                    sys.exit(1)

            # keep only nodes that were successfully started
            return set(node for node, ok
                       in itertools.izip(nodes, result.get()) if ok)
1271507
public static void main(String[] args) {
		// with or without - results on get/start/stop are virtually the same!
		ExampleUtils.fillManagerWithSimons(100000);

		StopwatchSample[] results = BenchmarkUtils.run(2, 5,
			new BenchmarkUtils.Task("empty") {
				@Override
				public void perform() throws Exception {
					//noinspection StatementWithEmptyBody
					for (int i = 0; i < LOOP; i++) {
					}
				}
			},
			new BenchmarkUtils.Task("millis") {
				@Override
				public void perform() throws Exception {
					for (int i = 0; i < LOOP; i++) {
						System.currentTimeMillis();
					}
				}
			},
			new BenchmarkUtils.Task("nanos") {
				@Override
				public void perform() throws Exception {
					for (int i = 0; i < LOOP; i++) {
						System.nanoTime();
					}
				}
			},
			new BenchmarkUtils.Task("assign-ms") {
				@Override
				public void perform() throws Exception {
					for (int i = 0; i < LOOP; i++) {
						long ms = System.currentTimeMillis();
					}
				}
			},
			new BenchmarkUtils.Task("assign-ns") {
				@Override
				public void perform() throws Exception {
					for (int i = 0; i < LOOP; i++) {
						long ns = System.nanoTime();
					}
				}
			},
			new BenchmarkUtils.Task("just-start") {
				@Override
				public void perform() throws Exception {
					Stopwatch simon = SimonManager.getStopwatch(null);
					for (int i = 0; i < LOOP; i++) {
						simon.start();
					}
				}
			},
			new BenchmarkUtils.Task("start-stop") {
				@Override
				public void perform() throws Exception {
					Stopwatch simon = SimonManager.getStopwatch(null);
					for (int i = 0; i < LOOP; i++) {
						simon.start().stop();
					}
				}
			},
			new BenchmarkUtils.Task("get-start-stop") {
				@Override
				public void perform() throws Exception {
					for (int i = 0; i < LOOP; i++) {
						SimonManager.getStopwatch("some.name").start().stop();
					}
				}
			}
		);

		System.out.println("\nGoogle Chart avg:\n" +
			GoogleChartImageGenerator.barChart("10M-loop duration", results));
		System.out.println("\nGoogle Chart avg/max/min:\n" +
			GoogleChartImageGenerator.barChart("10M-loop duration", results));
	}
686837
public void startGSEXIT() throws IOException {
    List<HubDataSink> dataSinks = hubDataStore.getHubDataSinks();
    for (HubDataSink hubDataSink : dataSinks) {
      List<String> com = Lists.newArrayList();
      com.add(getHostPort(hubDataStore.getHubAddress()));
      com.add(hubDataStore.getInstanceName());
      com.add(hubDataSink.getFTAName());
      com.add(hubDataSink.getName());
      ExternalProgramExecutor executorService = new ExternalProgramExecutor(
        "GSEXIT", new File(binLocation, "GSEXIT"), com.toArray(new String[com.size()]));
      gsExitExecutor.add(executorService);
      LOG.info("Starting GSEXIT : {}", executorService);
      executorService.startAndWait();
    }
  }
1302885
private void updateRunnability(ClusterNode node) {
    synchronized (node) {
      for (Map.Entry<ResourceType, RunnableIndices> entry :
        typeToIndices.entrySet()) {
        ResourceType type = entry.getKey();
        RunnableIndices r = entry.getValue();
        ResourceRequest unitReq = Utilities.getUnitResourceRequest(type);
        boolean currentlyRunnable = r.hasRunnable(node);
        boolean shouldBeRunnable = node.checkForGrant(unitReq, resourceLimit);
        if (currentlyRunnable && !shouldBeRunnable) {
          LOG.info("Node " + node.getName() + " is no longer " +
            type + " runnable");
          r.deleteRunnable(node);
        } else if (!currentlyRunnable && shouldBeRunnable) {
          LOG.info("Node " + node.getName() + " is now " + type + " runnable");
          r.addRunnable(node);
        }
      }
    }
  }
1025952
public void run() {
		ConnectionHandle connection = null;
		long tmp;
		long nextCheckInMs = this.maxAgeInMs;

		int partitionSize= this.partition.getAvailableConnections();
		long currentTime = System.currentTimeMillis();
		for (int i=0; i < partitionSize; i++){
			try {
				connection = this.partition.getFreeConnections().poll();

				if (connection != null){
					connection.setOriginatingPartition(this.partition);

					tmp = this.maxAgeInMs - (currentTime - connection.getConnectionCreationTimeInMs()); 

					if (tmp < nextCheckInMs){
						nextCheckInMs = tmp; 
					}

					if (connection.isExpired(currentTime)){
						// kill off this connection
						closeConnection(connection);
						continue;
					}


					
					if (this.lifoMode){
						// we can't put it back normally or it will end up in front again.
						if (!(connection.getOriginatingPartition().getFreeConnections().offer(connection))){
							connection.internalClose();
						}
					} else {
						this.pool.putConnectionBackInPartition(connection);
					}


					Thread.sleep(20L); // test slowly, this is not an operation that we're in a hurry to deal with (avoid CPU spikes)...
				}
			}  catch (Throwable e) {
					logger.error("Connection max age thread exception.", e);
			}

		} // throw it back on the queue

	}
1004332
private synchronized void configure(Configuration config) throws IOException {
    logger.atFine().log("GHFS.configure");
    logger.atFine().log("GHFS_ID = %s", GHFS_ID);

    overrideConfigFromFile(config);
    copyDeprecatedConfigurationOptions(config);
    // Set this configuration as the default config for this instance.
    setConf(config);

    enableFlatGlob = GCS_FLAT_GLOB_ENABLE.get(config, config::getBoolean);
    enableConcurrentGlob = GCS_CONCURRENT_GLOB_ENABLE.get(config, config::getBoolean);
    checksumType = GCS_FILE_CHECKSUM_TYPE.get(config, config::getEnum);
    defaultBlockSize = BLOCK_SIZE.get(config, config::getLong);
    reportedPermissions = new FsPermission(PERMISSIONS_TO_REPORT.get(config, config::get));

    if (gcsFsSupplier == null) {
      if (GCS_LAZY_INITIALIZATION_ENABLE.get(config, config::getBoolean)) {
        gcsFsSupplier =
            Suppliers.memoize(
                () -> {
                  try {
                    GoogleCloudStorageFileSystem gcsFs = createGcsFs(config);

                    pathCodec = gcsFs.getPathCodec();
                    configureBuckets(gcsFs);
                    configureWorkingDirectory(config);
                    gcsFsInitialized = true;

                    return gcsFs;
                  } catch (IOException e) {
                    throw new RuntimeException("Failed to create GCS FS", e);
                  }
                });
        pathCodec = getPathCodec(config);
      } else {
        setGcsFs(createGcsFs(config));
        configureBuckets(getGcsFs());
        configureWorkingDirectory(config);
      }
    } else {
      configureBuckets(getGcsFs());
      configureWorkingDirectory(config);
    }

    logger.atFine().log("GHFS.configure: done");
  }
812989
private static void locateForCouchbaseBucket(final BinaryRequest request, final List<Node> nodes,
        final CouchbaseBucketConfig config, CoreEnvironment env, RingBuffer<ResponseEvent> responseBuffer) {

        if (!keyIsValid(request)) {
            return;
        }

        int partitionId = partitionForKey(request.keyBytes(), config.numberOfPartitions());
        request.partition((short) partitionId);

        int nodeId = calculateNodeId(partitionId, request, config);
        if (nodeId < 0) {
            errorObservables(nodeId, request, config.name(), env, responseBuffer);
            return;
        }

        NodeInfo nodeInfo = config.nodeAtIndex(nodeId);

        for (Node node : nodes) {
            if (node.hostname().equals(nodeInfo.hostname())) {
                node.send(request);
                return;
            }
        }

        if(handleNotEqualNodeSizes(config.nodes().size(), nodes.size())) {
            RetryHelper.retryOrCancel(env, request, responseBuffer);
            return;
        }

        throw new IllegalStateException("Node not found for request" + request);
    }
1003336
@Override
    public void run(SiteProcedureConnection siteConnection)
    {
        hostLog.debug("STARTING: " + this);
        m_mailbox.send(com.google_voltpatches.common.primitives.Longs.toArray(m_initiatorHSIds), m_initiationMsg);
        m_txnState.setDone();
        m_queue.flush(getTxnId());
        execLog.l7dlog( Level.TRACE, LogKeys.org_voltdb_ExecutionSite_SendingCompletedWUToDtxn.name(), null);
        hostLog.debug("COMPLETE: " + this);
    }