Calculates the autocorrelation of a flatDataCollection for a predifined lag

@param flatDataList
@param lags
@return
544604
public static double autocorrelation(FlatDataList flatDataList, int lags) {
        int n = count(flatDataList);
        if(n<=0 || lags<=0 || n<lags) {
            throw new IllegalArgumentException("All the parameters must be positive and n larger than lags.");
        }
        
        FlatDataCollection flatDataCollection = flatDataList.toFlatDataCollection();
        double mean = mean(flatDataCollection);
        double variance = variance(flatDataCollection, true);
        
        double Ak=0.0;
        
        int maxI=n-lags;
        for(int i=0;i<maxI;++i) {
            Ak += (flatDataList.getDouble(i) - mean)*
                  (flatDataList.getDouble(i+lags) - mean);
        }
        
        Ak/=(n-lags);
        
        double autocorrelation = Ak/variance;
        
        return autocorrelation;
    }
1268263
public static double covariance(TransposeDataList transposeDataList, boolean isSample) {
        if(transposeDataList.size()!=2) {
            throw new IllegalArgumentException("The collection must contain observations from 2 groups.");
        }
        
        double covariance = 0.0;
        
        Object[] keys = transposeDataList.keySet().toArray();
        Object keyX = keys[0];
        Object keyY = keys[1];
        
        FlatDataCollection flatDataCollectionX = transposeDataList.get(keyX).toFlatDataCollection();
        FlatDataCollection flatDataCollectionY = transposeDataList.get(keyY).toFlatDataCollection();
        
        int n = flatDataCollectionX.size();
        if(n<=1 || n!=flatDataCollectionY.size()) {
            throw new IllegalArgumentException("The number of observations in each group must be equal and larger than 1.");
        }
        
        double meanX = mean(flatDataCollectionX);
        double meanY = mean(flatDataCollectionY);
        
        Iterator<Double> iteratorX = flatDataCollectionX.iteratorDouble();
        Iterator<Double> iteratorY = flatDataCollectionY.iteratorDouble();
        while (iteratorX.hasNext() && iteratorY.hasNext()) {
            covariance+=(iteratorX.next()-meanX)*
                        (iteratorY.next()-meanY);
        }
        
        if(isSample) {
            covariance/=(n-1.0);
        }
        else {
            covariance/=n;
        }
        
        return covariance;
    }
270829
def laggeddates(ts, step=1):
    ''''''
    if step == 1:
        dates = ts.dates()
        if not hasattr(dates, 'next'):
            dates = dates.__iter__()
        dt0 = next(dates)
        for dt1 in dates:
            yield dt1, dt0
            dt0 = dt1
    else:
        while done:
            done += 1
            lag.append(next(dates))
        for dt1 in dates:
            lag.append(dt1)
            yield dt1, lag.pop(0)
288840
private static FactorComparator<Executor> getLstDispatchedTimeComparator(final int weight) {
    return FactorComparator
        .create(LSTDISPATCHED_COMPARATOR_NAME, weight, new Comparator<Executor>() {

          @Override
          public int compare(final Executor o1, final Executor o2) {
            final ExecutorInfo stat1 = o1.getExecutorInfo();
            final ExecutorInfo stat2 = o2.getExecutorInfo();

            final int result = 0;
            if (statisticsObjectCheck(stat1, stat2, LSTDISPATCHED_COMPARATOR_NAME)) {
              return result;
            }
            // Note: an earlier date time indicates higher weight.
            return ((Long) stat2.getLastDispatchedTime()).compareTo(stat1.getLastDispatchedTime());
          }
        });
  }
170282
public static double xbarStd(TransposeDataCollection sampleDataCollection, AssociativeArray nh, AssociativeArray populationNh) {
        return Math.sqrt(xbarVariance(sampleDataCollection, nh, populationNh));
    }
754731
def _generate_lags(maxlag, multiplier):
    

    """
    # determine lag times
    lags = [1]
    # build default lag list
    lag = 1.0
    import decimal
    while lag <= maxlag:
        lag = lag*multiplier
        # round up, like python 2
        lag = int(decimal.Decimal(lag).quantize(decimal.Decimal('1'),
                                                rounding=decimal.ROUND_HALF_UP))
        if lag <= maxlag:
            ilag = int(lag)
            lags.append(ilag)
    # always include the maximal requested lag time.
    if maxlag not in lags:
        lags.append(maxlag)
    return np.array(lags)
705171
private static int aggregateDimValue(
      final int[] positions,
      final BufferAggregator[] theAggregators,
      final ByteBuffer resultsBuf,
      final int numBytesPerRecord,
      final int[] aggregatorOffsets,
      final int aggSize,
      final int aggExtra,
      final int dimIndex,
      int currentPosition
  )
  {
    if (SKIP_POSITION_VALUE == positions[dimIndex]) {
      return currentPosition;
    }
    if (INIT_POSITION_VALUE == positions[dimIndex]) {
      positions[dimIndex] = currentPosition * numBytesPerRecord;
      currentPosition++;
      final int pos = positions[dimIndex];
      for (int j = 0; j < aggSize; ++j) {
        theAggregators[j].init(resultsBuf, pos + aggregatorOffsets[j]);
      }
    }
    final int position = positions[dimIndex];

    switch (aggExtra) {
      case 7:
        theAggregators[6].aggregate(resultsBuf, position + aggregatorOffsets[6]);
        // fall through
      case 6:
        theAggregators[5].aggregate(resultsBuf, position + aggregatorOffsets[5]);
        // fall through
      case 5:
        theAggregators[4].aggregate(resultsBuf, position + aggregatorOffsets[4]);
        // fall through
      case 4:
        theAggregators[3].aggregate(resultsBuf, position + aggregatorOffsets[3]);
        // fall through
      case 3:
        theAggregators[2].aggregate(resultsBuf, position + aggregatorOffsets[2]);
        // fall through
      case 2:
        theAggregators[1].aggregate(resultsBuf, position + aggregatorOffsets[1]);
        // fall through
      case 1:
        theAggregators[0].aggregate(resultsBuf, position + aggregatorOffsets[0]);
    }
    for (int j = aggExtra; j < aggSize; j += AGG_UNROLL_COUNT) {
      theAggregators[j].aggregate(resultsBuf, position + aggregatorOffsets[j]);
      theAggregators[j + 1].aggregate(resultsBuf, position + aggregatorOffsets[j + 1]);
      theAggregators[j + 2].aggregate(resultsBuf, position + aggregatorOffsets[j + 2]);
      theAggregators[j + 3].aggregate(resultsBuf, position + aggregatorOffsets[j + 3]);
      theAggregators[j + 4].aggregate(resultsBuf, position + aggregatorOffsets[j + 4]);
      theAggregators[j + 5].aggregate(resultsBuf, position + aggregatorOffsets[j + 5]);
      theAggregators[j + 6].aggregate(resultsBuf, position + aggregatorOffsets[j + 6]);
      theAggregators[j + 7].aggregate(resultsBuf, position + aggregatorOffsets[j + 7]);
    }
    return currentPosition;
  }
830610
public double getElapsedTimeLastThread(int stage) {
	   if (stage == ProfilingEvent.START.ordinal()) {
           return 0;    
	   }
	   
	   Accumulator acc = lastAccumulator.get();

	   return acc == null ? Double.NaN : (acc.currentTimes[stage] - acc.currentTimes[stage - 1]) / MILLION;
   }
368161
private Double aggregate(Datasource ds, Collection<Double> values) {
        return ((values.size() / m_intervalsPer) > ds.getXff()) ? ds.getAggregationFuction().apply(values) : Double.NaN;
    }
46453
public static double calculateScore(FlatDataList errorList) {
        double DWdeltasquare=0;
        double DWetsquare=0;

        int n = errorList.size();
        for(int i=0;i<n;++i) {
            Double error = errorList.getDouble(i);
            if(i>=1) {
                Double errorPrevious = errorList.getDouble(i-1);
                if(errorPrevious!=null) {
                    DWdeltasquare+=Math.pow(error - errorPrevious,2);        
                }
            }
            DWetsquare+=error*error;
        }
        double DW=DWdeltasquare/DWetsquare;

        return DW;
    }
112945
public static double mean(FlatDataCollection flatDataCollection) {
        int n = 0;
        double mean = 0.0;
        Iterator<Double> it = flatDataCollection.iteratorDouble();
        while(it.hasNext()) {
            Double value = it.next();
            if(value != null) {
                ++n;
                mean += value;
            }
        }

        if(n==0) {
            throw new IllegalArgumentException("No not null values where found in the collection.");
        }
        mean /= n;
        
        return mean;
    }
1113131
public StatsImpl getStats(int[] dataIds, boolean recursive) {
        if (dataIds == null)
            return getStats(recursive);

        if (instance == null)
            return null;

        SpdData[] dataList = instance.listData(dataIds);
        ModuleItem[] items = children();
        ArrayList dataMembers = null; // return data members
        ArrayList colMembers = null; // return subcollection members

        // convert from Spd to Wpd and set dataMembers
        if (dataList != null) {
            dataMembers = new ArrayList(dataList.length);
            for (int i = 0; i < dataList.length; i++) {
                dataMembers.add(dataList[i].getStatistic());
            }
        }

        // getStats from children
        if (recursive && (items != null)) {
            colMembers = new ArrayList(items.length);
            for (int i = 0; i < items.length; i++) {
                colMembers.add(items[i].getStats(dataIds, recursive));
            }
        }

        // return Stats - individual instance will return different XXXStats instance
        // Note: construct dataMembers in ModuleItem and pass it to instance
        //       because getStats will be overwritten by subclasses
        //       and we don't want to duplicate the same code.
        return instance.getStats(dataMembers, colMembers);
    }
1247304
public static double getPvalue(TransposeDataCollection transposeDataCollection) {
        if(transposeDataCollection.size()!=2) {
            throw new IllegalArgumentException("The collection must contain observations from 2 groups.");
        }
        
        Object[] keys = transposeDataCollection.keySet().toArray();
        
        //IMPORTANT!!! place as n1 the smallest sample!!!!! This is required by the Test.
        
        
        int smallIndex = 0;
        int largeIndex = 1;
        int n1 = transposeDataCollection.get(keys[smallIndex]).size();
        int n2 = transposeDataCollection.get(keys[largeIndex]).size();
        if(n1>n2) {
            int tmp = n1;
            n1 = n2;
            n2 = tmp;
            smallIndex=1;
            //largeIndex=0;
        }
        
        //flatten the original internalData table
        AssociativeArray associativeArray = new AssociativeArray();
        
        for(Map.Entry<Object, FlatDataCollection> entry : transposeDataCollection.entrySet()) {
            Object i = entry.getKey();
            FlatDataCollection row = entry.getValue();
            
            Integer j=0;
            for (Object value : row) {
                Map.Entry<Object, Object> i_j = new AbstractMap.SimpleEntry<>(i, (Object)j);
                associativeArray.put(i_j, value);
                ++j;
            }
        }
        
        //converts the values of the flatDataCollection with their Ranks
        Ranks.getRanksFromValues(associativeArray);

        //sum up the scores of the smallest sample
        double MWscore=0.0;
        
        for(Map.Entry<Object,Object> entry : associativeArray.entrySet()) {
            @SuppressWarnings("unchecked")
            Map.Entry<Object, Object> i_j = (Map.Entry<Object, Object>)entry.getKey();
            Object i = i_j.getKey(); //get i and j values
            
            if(i.equals(keys[smallIndex])) { //if it belongs to the FIRST group (small group)
                Double rank = TypeInference.toDouble(entry.getValue());
                MWscore+=rank; //add the score
            }
        }
        //associativeArray=null;

        double pvalue= scoreToPvalue(MWscore, n1, n2);

        return pvalue;
    }
218141
@Override
  protected MtasDataItemDoubleFull getItem(int i) {
    if (i >= 0 && i < size) {
      return new MtasDataItemDoubleFull(
          ArrayUtils.toPrimitive(fullValueList[i]),
          hasSub() ? subCollectorListNextLevel[i] : null, getStatsItems(),
          sortType, sortDirection, errorNumber[i], errorList[i],
          sourceNumberList[i]);
    } else {
      return null;
    }
  }
177526
protected void aggregateCluster(ClusKernel otherData, long currentTime,
            double negLambda) {
        this.getData().aggregate(otherData, currentTime - this.timestamp,
                negLambda);
        this.timestamp = currentTime;
    }
927823
public static Axis generateAxisFromCollection(List<Float> axisValues) {
        List<AxisValue> values = new ArrayList<AxisValue>();
        int index = 0;
        for (float value : axisValues) {
            AxisValue axisValue = new AxisValue(value);
            values.add(axisValue);
            ++index;
        }

        Axis axis = new Axis(values);
        return axis;
    }
1169818
public static double getPvalue(TransposeDataCollection transposeDataCollection) {
        if(transposeDataCollection.size()!=2) {
            throw new IllegalArgumentException("The collection must contain observations from 2 groups.");
        }
        
        Object[] keys = transposeDataCollection.keySet().toArray();
        
        //counter of uncencored internalData in each group
        Map<Object, Integer> r = new HashMap<>();
        r.put(keys[0], 0);
        r.put(keys[1], 0);
        
        List<Object> mergedUncensoredData = new ArrayList<>();
        
        for(Map.Entry<Object, FlatDataCollection> entry : transposeDataCollection.entrySet()) {
            Object j = entry.getKey();
            FlatDataCollection flatDataCollection = entry.getValue();
            
            for(Object value : flatDataCollection) {
                if(!value.toString().endsWith(CensoredDescriptives.CENSORED_NUMBER_POSTFIX)) { //only the uncencored internalData
                    mergedUncensoredData.add(value);
                    r.put(j, r.get(j)+1);
                }
            }
        }
        Collections.sort(mergedUncensoredData, (Object o1, Object o2) -> {
            double v1 = TypeInference.toDouble(o1);
            double v2 = TypeInference.toDouble(o2);
            if(v1>v2) {
                return 1;
            }
            else if(v1<v2) {
                return -1;
            }
            return 0;
        });

        AssociativeArray2D testTable = new AssociativeArray2D();
        for(Object ti : mergedUncensoredData) {
            Double tiValue = TypeInference.toDouble(ti);
            Object value = testTable.get2d(ti, "mi");
            if(value==null) {
                testTable.put2d(ti, "mi", 1);
                testTable.put2d(ti, "r0ti", 0);
                testTable.put2d(ti, "r1ti", 0);
            }
            else {
                testTable.put2d(ti, "mi", ((Integer)value) +1); 
                continue; //continue in order not to count twice the r*ti below
            }

            //calculate the r*ti: the number of observations (both cencored and uncencored) which are larger than ti
            for(Map.Entry<Object, FlatDataCollection> entry : transposeDataCollection.entrySet()) {
                Object j = entry.getKey();
                FlatDataCollection flatDataCollection = entry.getValue();
                
                int jIndex = (j.equals(keys[1]))?1:0;
                
                //calculate the r*ti: the number of observations (both cencored and uncencored) which are larger than $ti
                String rJtiKey="r"+String.valueOf(jIndex)+"ti";
                for(Object value2 : flatDataCollection) {
                    
                    double v;
                    String str = value2.toString();
                    if(str.endsWith(CensoredDescriptives.CENSORED_NUMBER_POSTFIX)) {
                        //censored internalData encoded as 4.3+ or -4.3+
                        v = Double.valueOf(str.substring(0,str.length()-CensoredDescriptives.CENSORED_NUMBER_POSTFIX.length())); //remove the trailing char and convert it to double
                    }
                    else {
                        //uncensored internalData
                        v = TypeInference.toDouble(value2); //convert it to double
                    }
                    
                    if(v>=tiValue) {
                        testTable.put2d(ti, rJtiKey, (Integer)testTable.get2d(ti, rJtiKey) +1);
                    }
                }
            }
        }
        //mergedUncensoredData = null;

        double U = (double)r.get(keys[1]);
        double VarU =0.0;
        for(Map.Entry<Object, AssociativeArray> entry : testTable.entrySet()) {
            //Object key = entry.getKey();
            AssociativeArray testRow = entry.getValue();
            
            double r0ti=testRow.getDouble("r0ti");
            double r1ti=testRow.getDouble("r1ti");
            double mi = testRow.getDouble("mi");
            double rti= r0ti+r1ti;
            double Ai= r1ti/rti;
            U-=mi*Ai;

            VarU+=((mi*(rti-mi))/(rti-1.0))*Ai*(1.0-Ai);
        }
        //testTable = null;

        double Z=U/Math.sqrt(VarU);

        double pvalue = scoreToPvalue(Z);

        return pvalue;
    }