Filter data on thresholds of all available data on the tin.

<p><b>Note: At the first run of this method, only thresholds are calculated.</b></p>

@param lasHandler the las data handler of all data.
@throws Exception
189468
public void filterOnAllData( final ALasDataManager lasHandler ) throws Exception {
        final ConcurrentSkipListSet<Double> angleSet = new ConcurrentSkipListSet<Double>();
        final ConcurrentSkipListSet<Double> distanceSet = new ConcurrentSkipListSet<Double>();

        if (isFirstStatsCalculation) {
            pm.beginTask("Calculating initial statistics...", tinGeometries.length);
        } else {
            pm.beginTask("Filtering all data on seeds tin...", tinGeometries.length);
        }
        try {
            final List<Coordinate> newTotalLeftOverCoordinateList = new ArrayList<Coordinate>();
            if (threadsNum > 1) {
                // multithreaded
                ThreadedRunnable tRun = new ThreadedRunnable(threadsNum, null);
                for( final Geometry tinGeom : tinGeometries ) {
                    tRun.executeRunnable(new Runnable(){
                        public void run() {
                            List<Coordinate> leftOverList = runFilterOnAllData(lasHandler, angleSet, distanceSet, tinGeom);
                            synchronized (newTotalLeftOverCoordinateList) {
                                newTotalLeftOverCoordinateList.addAll(leftOverList);
                            }
                        }
                    });
                }
                tRun.waitAndClose();
            } else {
                for( final Geometry tinGeom : tinGeometries ) {
                    List<Coordinate> leftOverList = runFilterOnAllData(lasHandler, angleSet, distanceSet, tinGeom);
                    newTotalLeftOverCoordinateList.addAll(leftOverList);
                }
            }
            pm.done();

            leftOverCoordinateList.clear();
            leftOverCoordinateList.addAll(newTotalLeftOverCoordinateList);

            /*
             * now recalculate the thresholds
             */
            if (angleSet.size() > 1) {
                calculatedAngleThreshold = getMedianFromSet(angleSet);
                pm.message("Calculated angle threshold: " + calculatedAngleThreshold + " (range: " + angleSet.first() + " to "
                        + angleSet.last() + ")");
            } else if (angleSet.size() == 0) {
                return;
            } else {
                calculatedAngleThreshold = angleSet.first();
                pm.message("Single angle left: " + calculatedAngleThreshold);
            }

            if (distanceSet.size() > 1) {
                calculatedDistanceThreshold = getMedianFromSet(distanceSet);
                pm.message("Calculated distance threshold: " + calculatedDistanceThreshold + " (range: " + distanceSet.first()
                        + " to " + distanceSet.last() + ")");
            } else if (distanceSet.size() == 0) {
                return;
            } else {
                calculatedDistanceThreshold = distanceSet.first();
                pm.message("Single distance left: " + calculatedDistanceThreshold);
            }
            if (isFirstStatsCalculation) {
                isFirstStatsCalculation = false;
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
62345
public Iterator<?> getElements(Filter filter, int offset, int maxResultSize) throws LayerException {
		if (null == filter) {
			filter = Filter.INCLUDE;
		}
		List<Object> filteredList = new ArrayList<Object>();
		try {
			synchronized (featuresById) {
				for (Object feature : featuresById.values()) {
					if (filter.evaluate(feature)) {
						filteredList.add(feature);
					}
				}
			}
		} catch (Exception e) { // NOSONAR
			throw new LayerException(e, ExceptionCode.FILTER_EVALUATION_PROBLEM, filter, getId());
		}
		// Sorting of elements.
		if (comparator != null) {
			Collections.sort(filteredList, comparator);
		}
		if (maxResultSize > 0) {
			int fromIndex = Math.max(0, offset);
			int toIndex = Math.min(offset + maxResultSize, filteredList.size());
			toIndex = Math.max(fromIndex, toIndex);
			return filteredList.subList(fromIndex, toIndex).iterator();
		} else {
			return filteredList.iterator();
		}
	}
990464
public void dropElementsBelowThreshold(double threshold) {
        Iterator<T> iterator = keySet().iterator();
        while (iterator.hasNext()) {
            T element  = iterator.next();
            double val = map.get(element).get();
            if (val < threshold) {
                iterator.remove();
                dirty.set(true);
            }
        }

    }
204069
public void setWithTotal() throws IOException {
    if (collectorType.equals(DataCollector.COLLECTOR_TYPE_LIST)) {
      if (segmentName != null) {
        throw new IOException("can't get total with segmentRegistration");
      } else {
        withTotal = true;
      }
    } else {
      throw new IOException(
          "can't get total for dataCollector of type " + collectorType);
    }
  }
707547
protected void checkMissing(Instances instances) throws Exception {
    for (int i = 0; i < instances.numInstances(); i++) {
      Instance ins = instances.instance(i);
      for (int j = 0; j < ins.numValues(); j++) {
        if (ins.index(j) != ins.classIndex())
          if (ins.isMissingSparse(j)) {
            throw new Exception("ERROR: KDTree can not deal with missing "
                + "values. Please run ReplaceMissingValues filter "
                + "on the dataset before passing it on to the KDTree.");
          }
      }
    }
  }
361158
public <E extends Exception> boolean noneMatch(Try.BooleanPredicate<E> filter) throws E {
        return noneMatch(0, size(), filter);
    }
593012
public static void setHandler(XMLReader reader, Object handler) throws SAXNotSupportedException, SAXNotRecognizedException{
        if(handler instanceof ContentHandler)
            reader.setContentHandler((ContentHandler)handler);
        if(handler instanceof EntityResolver)
            reader.setEntityResolver((EntityResolver)handler);
        if(handler instanceof ErrorHandler)
            reader.setErrorHandler((ErrorHandler)handler);
        if(handler instanceof DTDHandler)
            reader.setDTDHandler((DTDHandler)handler);
        if(handler instanceof LexicalHandler){
            try{
                reader.setProperty(SAXProperties.LEXICAL_HANDLER, handler);
            }catch(SAXException ex){
                reader.setProperty(SAXProperties.LEXICAL_HANDLER_ALT, handler);
            }
        }
        if(handler instanceof DeclHandler){
            try{
                reader.setProperty(SAXProperties.DECL_HANDLER, handler);
            }catch(SAXException ex){
                reader.setProperty(SAXProperties.DECL_HANDLER_ALT, handler);
            }
        }
    }
140468
public LambdaDslObject and(String name, Object value, MatchingRule... rules) {
        object.and(name, value, rules);
        return this;
    }
1304952
public void runFetcher() throws Exception {

		// check that we are running before proceeding
		if (!running) {
			return;
		}

		this.mainThread = Thread.currentThread();

		// ------------------------------------------------------------------------
		//  Procedures before starting the infinite while loop:
		// ------------------------------------------------------------------------

		//  1. check that there is at least one shard in the subscribed streams to consume from (can be done by
		//     checking if at least one value in subscribedStreamsToLastDiscoveredShardIds is not null)
		boolean hasShards = false;
		StringBuilder streamsWithNoShardsFound = new StringBuilder();
		for (Map.Entry<String, String> streamToLastDiscoveredShardEntry : subscribedStreamsToLastDiscoveredShardIds.entrySet()) {
			if (streamToLastDiscoveredShardEntry.getValue() != null) {
				hasShards = true;
			} else {
				streamsWithNoShardsFound.append(streamToLastDiscoveredShardEntry.getKey()).append(", ");
			}
		}

		if (streamsWithNoShardsFound.length() != 0 && LOG.isWarnEnabled()) {
			LOG.warn("Subtask {} has failed to find any shards for the following subscribed streams: {}",
				indexOfThisConsumerSubtask, streamsWithNoShardsFound.toString());
		}

		if (!hasShards) {
			throw new RuntimeException("No shards can be found for all subscribed streams: " + streams);
		}

		//  2. start consuming any shard state we already have in the subscribedShardState up to this point; the
		//     subscribedShardState may already be seeded with values due to step 1., or explicitly added by the
		//     consumer using a restored state checkpoint
		for (int seededStateIndex = 0; seededStateIndex < subscribedShardsState.size(); seededStateIndex++) {
			KinesisStreamShardState seededShardState = subscribedShardsState.get(seededStateIndex);

			// only start a consuming thread if the seeded subscribed shard has not been completely read already
			if (!seededShardState.getLastProcessedSequenceNum().equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) {

				if (LOG.isInfoEnabled()) {
					LOG.info("Subtask {} will start consuming seeded shard {} from sequence number {} with ShardConsumer {}",
						indexOfThisConsumerSubtask, seededShardState.getStreamShardHandle().toString(),
						seededShardState.getLastProcessedSequenceNum(), seededStateIndex);
					}

				shardConsumersExecutor.submit(
					createShardConsumer(
						seededStateIndex,
						subscribedShardsState.get(seededStateIndex).getStreamShardHandle(),
						subscribedShardsState.get(seededStateIndex).getLastProcessedSequenceNum(),
						registerShardMetrics(consumerMetricGroup, subscribedShardsState.get(seededStateIndex))));
			}
		}

        // start periodic watermark emitter, if a watermark assigner was configured
		if (periodicWatermarkAssigner != null) {
			long periodicWatermarkIntervalMillis = runtimeContext.getExecutionConfig().getAutoWatermarkInterval();
			if (periodicWatermarkIntervalMillis > 0) {
				ProcessingTimeService timerService = ((StreamingRuntimeContext) runtimeContext).getProcessingTimeService();
				LOG.info("Starting periodic watermark emitter with interval {}", periodicWatermarkIntervalMillis);
				new PeriodicWatermarkEmitter(timerService, periodicWatermarkIntervalMillis).start();
			}
			this.shardIdleIntervalMillis = Long.parseLong(
				getConsumerConfiguration().getProperty(ConsumerConfigConstants.SHARD_IDLE_INTERVAL_MILLIS,
					Long.toString(ConsumerConfigConstants.DEFAULT_SHARD_IDLE_INTERVAL_MILLIS)));
		}

		// ------------------------------------------------------------------------

		// finally, start the infinite shard discovery and consumer launching loop;
		// we will escape from this loop only when shutdownFetcher() or stopWithError() is called

		final long discoveryIntervalMillis = Long.valueOf(
			configProps.getProperty(
				ConsumerConfigConstants.SHARD_DISCOVERY_INTERVAL_MILLIS,
				Long.toString(ConsumerConfigConstants.DEFAULT_SHARD_DISCOVERY_INTERVAL_MILLIS)));

		if (this.numberOfActiveShards.get() == 0) {
			LOG.info("Subtask {} has no active shards to read on startup; marking the subtask as temporarily idle ...",
				indexOfThisConsumerSubtask);
			sourceContext.markAsTemporarilyIdle();
		}

		while (running) {
			if (LOG.isDebugEnabled()) {
				LOG.debug("Subtask {} is trying to discover new shards that were created due to resharding ...",
					indexOfThisConsumerSubtask);
			}
			List<StreamShardHandle> newShardsDueToResharding = discoverNewShardsToSubscribe();

			for (StreamShardHandle shard : newShardsDueToResharding) {
				// since there may be delay in discovering a new shard, all new shards due to
				// resharding should be read starting from the earliest record possible
				KinesisStreamShardState newShardState =
					new KinesisStreamShardState(convertToStreamShardMetadata(shard), shard, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get());
				int newStateIndex = registerNewSubscribedShardState(newShardState);

				if (LOG.isInfoEnabled()) {
					LOG.info("Subtask {} has discovered a new shard {} due to resharding, and will start consuming " +
							"the shard from sequence number {} with ShardConsumer {}",
						indexOfThisConsumerSubtask, newShardState.getStreamShardHandle().toString(),
						newShardState.getLastProcessedSequenceNum(), newStateIndex);
				}

				shardConsumersExecutor.submit(
					createShardConsumer(
						newStateIndex,
						newShardState.getStreamShardHandle(),
						newShardState.getLastProcessedSequenceNum(),
						registerShardMetrics(consumerMetricGroup, newShardState)));
			}

			// we also check if we are running here so that we won't start the discovery sleep
			// interval if the running flag was set to false during the middle of the while loop
			if (running && discoveryIntervalMillis != 0) {
				try {
					Thread.sleep(discoveryIntervalMillis);
				} catch (InterruptedException iex) {
					// the sleep may be interrupted by shutdownFetcher()
				}
			}
		}

		// make sure all resources have been terminated before leaving
		try {
			awaitTermination();
		} catch (InterruptedException ie) {
			// If there is an original exception, preserve it, since that's more important/useful.
			this.error.compareAndSet(null, ie);
		}

		// any error thrown in the shard consumer threads will be thrown to the main thread
		Throwable throwable = this.error.get();
		if (throwable != null) {
			if (throwable instanceof Exception) {
				throw (Exception) throwable;
			} else if (throwable instanceof Error) {
				throw (Error) throwable;
			} else {
				throw new Exception(throwable);
			}
		}
	}
437962
public void setUserDetectorThreshold(int threshold) {
        String minPriority = ProjectFilterSettings.getIntPriorityAsString(threshold);
        filterSettings.setMinPriority(minPriority);
    }
616205
public void init() throws Exception {
		filter = bundleContext.createFilter(getFilter(dsName));
		
		tracker = new ServiceTracker<DataSource, DataSource>(bundleContext, filter, null);
		tracker.open();
		
		proxy = (DataSource) Proxy.newProxyInstance(
				DataSourceProxyFactory.class.getClassLoader(),
				new Class<?>[] { DataSource.class }, this);
	}
877491
final public void And() throws ParseException {
    Equality();
    label_8:
    while (true) {
      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
      case AND0:
      case AND1:
        ;
        break;
      default:
        jj_la1[12] = jj_gen;
        break label_8;
      }
      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
      case AND0:
        jj_consume_token(AND0);
        break;
      case AND1:
        jj_consume_token(AND1);
        break;
      default:
        jj_la1[13] = jj_gen;
        jj_consume_token(-1);
        throw new ParseException();
      }
                                  AstAnd jjtn001 = new AstAnd(JJTAND);
                                  boolean jjtc001 = true;
                                  jjtree.openNodeScope(jjtn001);
      try {
        Equality();
      } catch (Throwable jjte001) {
                                  if (jjtc001) {
                                    jjtree.clearNodeScope(jjtn001);
                                    jjtc001 = false;
                                  } else {
                                    jjtree.popNode();
                                  }
                                  if (jjte001 instanceof RuntimeException) {
                                    {if (true) throw (RuntimeException)jjte001;}
                                  }
                                  if (jjte001 instanceof ParseException) {
                                    {if (true) throw (ParseException)jjte001;}
                                  }
                                  {if (true) throw (Error)jjte001;}
      } finally {
                                  if (jjtc001) {
                                    jjtree.closeNodeScope(jjtn001,  2);
                                  }
      }
    }
  }
117785
protected RuntimeException runtimeException(final Exception exception) {
        if (exception instanceof RuntimeException) {
            throw (RuntimeException) exception;
        } else {
            throw new RuntimeException(exception);
        }
    }
645282
@Override
    public boolean checkCondition(IValue test) throws FilterException {
        Iterator iter = values.iterator();
        while (iter.hasNext()) {
            IValue value = (IValue) iter.next();
            if (value.containedBy(test)) {
                return true;
            }
        }
        return false;
    }
804853
public RunList<R> overThresholdOnly(final Result threshold) {
        return filter(new Predicate<R>() {
            public boolean apply(R r) {
                return (r.getResult() != null && r.getResult().isBetterOrEqualTo(threshold));
            }
        });
    }
226201
private void setThresholdAttributes(List<String> thresholdIds, HttpServletRequest request) throws APIException {
        if (thresholdIds == null || thresholdIds.isEmpty()) {
            return;
        }

        request.setAttribute("thresholdsPresent", Boolean.TRUE);
        List<ThresholdStatusBean> thresholdStatusBeans = getThresholdBeans(getThresholdAPI().getThresholdStatuses(thresholdIds.toArray(new String[0])));
        request.setAttribute("thresholds", thresholdStatusBeans);
    }
1210995
@Beta
    public <R, E extends Exception, E2 extends Exception> void filterThenForEach(final Try.Predicate<? super T, E> filter,
            final Try.Consumer<? super T, E2> action) throws E, E2 {
        N.checkArgNotNull(filter);
        N.checkArgNotNull(action);

        for (T e : coll) {
            if (filter.test(e)) {
                action.accept(e);
            }
        }
    }