Preprocess the dataset, precomputing the parameterization functions.

@param vrel Vector relation
@return Preprocessed relation
548891
private Relation<ParameterizationFunction> preprocess(Relation<V> vrel) {
    DBIDs ids = vrel.getDBIDs();
    SimpleTypeInformation<ParameterizationFunction> type = new SimpleTypeInformation<>(ParameterizationFunction.class);
    WritableDataStore<ParameterizationFunction> prep = DataStoreUtil.makeStorage(ids, DataStoreFactory.HINT_HOT, ParameterizationFunction.class);

    // Project
    for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
      prep.put(iter, new ParameterizationFunction(vrel.get(iter)));
    }
    return new MaterializedRelation<>(type, ids, null, prep);
  }
1025722
@Override
  public final List<Node<String>> getSampleDataRow(
    final Map<String, Object> pReqVars) throws Exception {
    String lang = (String) pReqVars.get("lang");
    List<Node<String>> result = new ArrayList<Node<String>>();
    Integer idx = 1;
    Node<String> nodeService = new Node<String>();
    result.add(nodeService);
    nodeService.setItsName(getSrvI18n().getMsg("service", lang));
    nodeService.setItsNodes(new ArrayList<Node<String>>());
    Node<String> nodeServiceName = new Node<String>();
    nodeService.getItsNodes().add(nodeServiceName);
    nodeServiceName.setItsName(getSrvI18n().getMsg("itsName", lang));
    nodeServiceName.setItsValue(idx.toString() + ";itsName");
    Node<String> nodeServiceId = new Node<String>();
    nodeService.getItsNodes().add(nodeServiceId);
    nodeServiceId.setItsName(getSrvI18n().getMsg("itsId", lang));
    nodeServiceId.setItsValue(idx.toString() + ";itsId");
    Node<String> nodeServiceItsCategory = new Node<String>();
    nodeService.getItsNodes().add(nodeServiceItsCategory);
    nodeServiceItsCategory.setItsName(getSrvI18n().getMsg("itsCategory", lang));
    nodeServiceItsCategory.setItsNodes(new ArrayList<Node<String>>());
    Node<String> nodeServiceItsCategoryName = new Node<String>();
    nodeServiceItsCategory.getItsNodes().add(nodeServiceItsCategoryName);
    nodeServiceItsCategoryName.setItsName(getSrvI18n().getMsg("itsName", lang));
    nodeServiceItsCategoryName
      .setItsValue(idx.toString() + ";itsCategory,itsName");
    Node<String> nodeServiceItsCategoryId = new Node<String>();
    nodeServiceItsCategory.getItsNodes().add(nodeServiceItsCategoryId);
    nodeServiceItsCategoryId.setItsName(getSrvI18n().getMsg("itsId", lang));
    nodeServiceItsCategoryId.setItsValue(idx.toString() + ";itsCategory,itsId");
    idx++;
    Node<String> nodePrice = new Node<String>();
    result.add(nodePrice);
    nodePrice.setItsName(getSrvI18n().getMsg("itsPrice", lang));
    nodePrice.setItsValue(idx.toString());
    idx++;
    Node<String> nodeCost = new Node<String>();
    result.add(nodeCost);
    nodeCost.setItsName(getSrvI18n().getMsg("itsCost", lang));
    nodeCost.setItsValue(idx.toString());
    idx++;
    Node<String> nodeQuantity = new Node<String>();
    result.add(nodeQuantity);
    nodeQuantity.setItsName(getSrvI18n().getMsg("itsQuantity", lang));
    nodeQuantity.setItsValue(idx.toString());
    idx++;
    Node<String> nodeIsAvailable = new Node<String>();
    result.add(nodeIsAvailable);
    nodeIsAvailable.setItsName(getSrvI18n().getMsg("isAvailable", lang));
    nodeIsAvailable.setItsValue(idx.toString());
    Set<String> ndFlIdNm = new HashSet<String>();
    ndFlIdNm.add("itsId");
    ndFlIdNm.add("itsName");
    pReqVars.put("InvItemTaxCategoryneededFields", ndFlIdNm);
    pReqVars.put("TaxneededFields", ndFlIdNm);
    List<InvItemTaxCategoryLine> allTaxCatsLns = getSrvOrm()
      .retrieveList(pReqVars, InvItemTaxCategoryLine.class);
    pReqVars.remove("InvItemTaxCategoryneededFields");
    pReqVars.remove("TaxneededFields");
    List<Tax> usedTaxes = new ArrayList<Tax>();
    List<InvItemTaxCategory> usedTaxCats = new ArrayList<InvItemTaxCategory>();
    for (InvItemTaxCategoryLine tcl : allTaxCatsLns) {
      boolean txListed = false;
      for (Tax tx : usedTaxes) {
        if (tx.getItsId().equals(tcl.getTax().getItsId())) {
          txListed = true;
          break;
        }
      }
      if (!txListed) {
        usedTaxes.add(tcl.getTax());
      }
      int tci = -1;
      for (InvItemTaxCategory tc : usedTaxCats) {
        if (tc.getItsId().equals(tcl.getItsOwner().getItsId())) {
          tci = usedTaxCats.indexOf(tc);
          break;
        }
      }
      if (tci == -1) {
        usedTaxCats.add(tcl.getItsOwner());
        tcl.getItsOwner().setTaxes(new ArrayList<InvItemTaxCategoryLine>());
        tcl.getItsOwner().getTaxes().add(tcl);
      } else {
        usedTaxCats.get(tci).getTaxes().add(tcl);
      }
    }
    boolean isOnlyTax = true;
    for (InvItemTaxCategory txc : usedTaxCats) {
      if (txc.getTaxes().size() > 1) {
        isOnlyTax = false;
        break;
      }
    }
    if (isOnlyTax) {
      idx++;
      addTaxWr(result, idx.toString(),
        getSrvI18n().getMsg("OnlyTax", lang), lang);
    } else {
      idx++;
      addTaxCatWr(result, idx.toString(),
        getSrvI18n().getMsg("taxCategory", lang), lang);
      Collections.sort(usedTaxes, new CmprHasIdLong<Tax>());
      for (Tax tx : usedTaxes) {
        idx++;
        addTaxWr(result, idx.toString(), tx.getItsName(), lang);
      }
      Collections.sort(usedTaxCats, new CmprHasIdLong<InvItemTaxCategory>());
      for (InvItemTaxCategory txc : usedTaxCats) {
        idx++;
        addTaxCatWr(result, idx.toString(), txc.getItsName(), lang);
      }
    }
    return result;
  }
36736
public void preSearch(Instances data) throws Exception {
    m_Cache        = new PerformanceCache();
    m_Performances = new Vector<Performance>();
    m_Trace        = new ArrayList<Entry<Integer, Performance>>();

    m_Owner.getGenerator().reset();
    m_Space = m_Owner.getGenerator().getSpace();
  }
238367
public function prepare(Mapper $mapper)
    {
        if ($this->isPrepared()) {
            throw new DataSetPreparationException('DataSet is already prepared');
        }
        //Split each row of the dataSet into inputs and outputs based on the mapper and the policies
        //Then re-parse the dataSet to fill in all the fields that used special policies (like Avg) and create the instances
        $this->map($mapper)->createInstances();

        //mark the dataSet as prepared
        $this->isPrepared = true;
    }
493315
GeometryCursor prepareVector_(VertexDescription descr, int dimensionMask,
			Geometry[] res_vec) {
		int inext = 0;
		if ((dimensionMask & 1) != 0) {
			if (res_vec[0] == null)
				res_vec[0] = new MultiPoint(descr);
			inext++;
		} else {
			for (int i = 0; i < res_vec.length - 1; i++)
				res_vec[i] = res_vec[i + 1];
		}

		if ((dimensionMask & 2) != 0) {
			if (res_vec[inext] == null)
				res_vec[inext] = new Polyline(descr);
			inext++;
		} else {
			for (int i = inext; i < res_vec.length - 1; i++)
				res_vec[i] = res_vec[i + 1];
		}

		if ((dimensionMask & 4) != 0) {
			if (res_vec[inext] == null)
				res_vec[inext] = new Polygon(descr);
			inext++;
		} else {
			for (int i = inext; i < res_vec.length - 1; i++)
				res_vec[i] = res_vec[i + 1];
		}
		
		if (inext != 3) {
			Geometry[] r = new Geometry[inext];
			for (int i = 0; i < inext; i++)
				r[i] = res_vec[i];

			return new SimpleGeometryCursor(r);
		} else {
			return new SimpleGeometryCursor(res_vec);
		}
	}
207953
public static <V extends SpatialComparable> String getColumnLabel(Relation<? extends V> rel, int col) {
    SimpleTypeInformation<? extends V> type = rel.getDataTypeInformation();
    if(!(type instanceof VectorFieldTypeInformation)) {
      return "Column " + col;
    }
    final VectorFieldTypeInformation<?> vtype = (VectorFieldTypeInformation<?>) type;
    String lbl = vtype.getLabel(col);
    return (lbl != null) ? lbl : ("Column " + col);
  }
545051
protected Set<String> getPrevExpandedRows() {
		Set<String> prev = getComponentModel().prevExpandedRows;
		if (prev == null) {
			return Collections.emptySet();
		} else {
			return Collections.unmodifiableSet(prev);
		}
	}
283762
@Override
    public boolean beforeSolve(ReconfigurationProblem p) throws SchedulerException {
        for (VM vm : source.getMapping().getAllVMs()) {
            int vmId = p.getVM(vm);
            int v = getVMAllocation(vmId);
            if (v < 0) {
                int prevUsage = rc.getConsumption(vm);
                minVMAllocation(vmId, prevUsage);
            }

        }

        ChocoView v = rp.getView(Packing.VIEW_ID);
        if (v == null) {
            throw SchedulerModelingException.missingView(rp.getSourceModel(), Packing.VIEW_ID);
        }

        IntVar[] host = new IntVar[p.getFutureRunningVMs().size()];
        int[] demand = new int[host.length];
        int i = 0;
        for (VM vm : p.getFutureRunningVMs()) {
            host[i] = rp.getVMAction(vm).getDSlice().getHoster();
            demand[i] = getVMAllocation(p.getVM(vm));
            i++;
        }
        ((Packing) v).addDim(rc.getResourceIdentifier(),
                virtRcUsage,
                demand,
                host);

        return linkVirtualToPhysicalUsage();
    }
758169
protected function preProcess(VcsDriverInterface $driver, array $data, $identifier)
    {
        $vcsRepos = array();
        $data = array_merge($data, $this->packageData);
        $data = $this->assetType->getPackageConverter()->convert($data, $vcsRepos);

        $this->addRepositories($vcsRepos);

        if (!isset($data['dist'])) {
            $data['dist'] = $driver->getDist($identifier);
        }
        if (!isset($data['source'])) {
            $data['source'] = $driver->getSource($identifier);
        }

        return $this->assetRepositoryManager->solveResolutions((array) $data);
    }
979645
@SuppressWarnings("unchecked")
    public static Map<String,TernaryVector> load(File indexVectorFile) {
        try {
            FileInputStream fis = new FileInputStream(indexVectorFile);
            ObjectInputStream inStream = new ObjectInputStream(fis);
            Map<String, TernaryVector> vectorMap = 
                (Map<String, TernaryVector>) inStream.readObject();
            inStream.close();
            return vectorMap;
        } catch (IOException ioe) {
            throw new IOError(ioe);
        } catch (ClassNotFoundException cnfe) {
            throw new Error(cnfe);
        }
    }
139995
public static void recomputeCovarianceMatrices(Relation<? extends NumberVector> relation, WritableDataStore<double[]> probClusterIGivenX, List<? extends EMClusterModel<?>> models, double prior) {
    final int k = models.size();
    boolean needsTwoPass = false;
    for(EMClusterModel<?> m : models) {
      m.beginEStep();
      needsTwoPass |= m.needsTwoPass();
    }
    // First pass, only for two-pass models.
    if(needsTwoPass) {
      for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) {
        double[] clusterProbabilities = probClusterIGivenX.get(iditer);
        NumberVector instance = relation.get(iditer);
        for(int i = 0; i < clusterProbabilities.length; i++) {
          final double prob = clusterProbabilities[i];
          if(prob > 1e-10) {
            models.get(i).firstPassE(instance, prob);
          }
        }
      }
      for(EMClusterModel<?> m : models) {
        m.finalizeFirstPassE();
      }
    }
    double[] wsum = new double[k];
    for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) {
      double[] clusterProbabilities = probClusterIGivenX.get(iditer);
      NumberVector instance = relation.get(iditer);
      for(int i = 0; i < clusterProbabilities.length; i++) {
        final double prob = clusterProbabilities[i];
        if(prob > 1e-10) {
          models.get(i).updateE(instance, prob);
        }
        wsum[i] += prob;
      }
    }
    for(int i = 0; i < models.size(); i++) {
      // MLE / MAP
      final double weight = prior <= 0. ? wsum[i] / relation.size() : (wsum[i] + prior - 1) / (relation.size() + prior * k - k);
      models.get(i).finalizeEStep(weight, prior);
    }
  }
203154
void preProcess(File osmFile) {
        try (OSMInput in = openOsmInputFile(osmFile)) {
            long tmpWayCounter = 1;
            long tmpRelationCounter = 1;
            ReaderElement item;
            while ((item = in.getNext()) != null) {
                if (item.isType(ReaderElement.WAY)) {
                    final ReaderWay way = (ReaderWay) item;
                    boolean valid = filterWay(way);
                    if (valid) {
                        LongIndexedContainer wayNodes = way.getNodes();
                        int s = wayNodes.size();
                        for (int index = 0; index < s; index++) {
                            prepareHighwayNode(wayNodes.get(index));
                        }

                        if (++tmpWayCounter % 10_000_000 == 0) {
                            LOGGER.info(nf(tmpWayCounter) + " (preprocess), osmIdMap:" + nf(getNodeMap().getSize()) + " ("
                                    + getNodeMap().getMemoryUsage() + "MB) " + Helper.getMemInfo());
                        }
                    }
                } else if (item.isType(ReaderElement.RELATION)) {
                    final ReaderRelation relation = (ReaderRelation) item;
                    if (!relation.isMetaRelation() && relation.hasTag("type", "route"))
                        prepareWaysWithRelationInfo(relation);

                    if (relation.hasTag("type", "restriction"))
                        prepareRestrictionRelation(relation);

                    if (++tmpRelationCounter % 100_000 == 0) {
                        LOGGER.info(nf(tmpRelationCounter) + " (preprocess), osmWayMap:" + nf(getRelFlagsMap().size())
                                + " " + Helper.getMemInfo());
                    }
                } else if (item.isType(ReaderElement.FILEHEADER)) {
                    final OSMFileHeader fileHeader = (OSMFileHeader) item;
                    osmDataDate = Helper.createFormatter().parse(fileHeader.getTag("timestamp"));
                }

            }
        } catch (Exception ex) {
            throw new RuntimeException("Problem while parsing file", ex);
        }
    }
1062680
@Override
    public void preProcess(@NonNull DataSet toPreProcess) {
        transform(toPreProcess.getFeatures(), toPreProcess.getFeaturesMaskArray());
        transformLabel(toPreProcess.getLabels(), toPreProcess.getLabelsMaskArray());
    }
477409
public double evaluateClustering(Database db, Relation<O> rel, Clustering<?> cl) {
    final DistanceQuery<O> dq = rel.getDistanceQuery(distanceFunction);

    List<? extends Cluster<?>> clusters = cl.getAllClusters();
    final int numc = clusters.size();

    // DBCV needs a "dimensionality".
    @SuppressWarnings("unchecked")
    final Relation<? extends SpatialComparable> vrel = (Relation<? extends SpatialComparable>) rel;
    final int dim = RelationUtil.dimensionality(vrel);

    // precompute all core distances
    ArrayDBIDs[] cids = new ArrayDBIDs[numc];
    double[][] coreDists = new double[numc][];
    for(int c = 0; c < numc; c++) {
      Cluster<?> cluster = clusters.get(c);
      // Singletons are considered as Noise, because they have no sparseness
      if(cluster.isNoise() || cluster.size() < 2) {
        coreDists[c] = null;
        continue;
      }
      // Store for use below:
      ArrayDBIDs ids = cids[c] = DBIDUtil.ensureArray(cluster.getIDs());
      double[] clusterCoreDists = coreDists[c] = new double[ids.size()];
      for(DBIDArrayIter it = ids.iter(), it2 = ids.iter(); it.valid(); it.advance()) {
        double currentCoreDist = 0;
        int neighbors = 0;
        for(it2.seek(0); it2.valid(); it2.advance()) {
          if(DBIDUtil.equal(it, it2)) {
            continue;
          }
          double dist = dq.distance(it, it2);
          // Unfortunately, the DBCV definition has a division by zero.
          // We ignore such objects.
          if(dist > 0) {
            currentCoreDist += MathUtil.powi(1. / dist, dim);
            ++neighbors;
          }
        }
        // Average, and undo power.
        clusterCoreDists[it.getOffset()] = FastMath.pow(currentCoreDist / neighbors, -1. / dim);
      }
    }

    // compute density sparseness of all clusters
    int[][] clusterDegrees = new int[numc][];
    double[] clusterDscMax = new double[numc];
    // describes if a cluster contains any internal edges
    boolean[] internalEdges = new boolean[numc];
    for(int c = 0; c < numc; c++) {
      Cluster<?> cluster = clusters.get(c);
      if(cluster.isNoise() || cluster.size() < 2) {
        clusterDegrees[c] = null;
        clusterDscMax[c] = Double.NaN;
        continue;
      }
      double[] clusterCoreDists = coreDists[c];
      ArrayDBIDs ids = cids[c];
      double dscMax = 0; // Density Sparseness of the Cluster
      double[][] distances = new double[cluster.size()][cluster.size()];

      // create mutability distance matrix for Minimum Spanning Tree
      for(DBIDArrayIter it = ids.iter(), it2 = ids.iter(); it.valid(); it.advance()) {
        double currentCoreDist = clusterCoreDists[it.getOffset()];
        for(it2.seek(it.getOffset() + 1); it2.valid(); it2.advance()) {
          double mutualReachDist = MathUtil.max(currentCoreDist, clusterCoreDists[it2.getOffset()], dq.distance(it, it2));
          distances[it.getOffset()][it2.getOffset()] = mutualReachDist;
          distances[it2.getOffset()][it.getOffset()] = mutualReachDist;
        }
      }

      // generate Minimum Spanning Tree
      int[] nodes = PrimsMinimumSpanningTree.processDense(distances);

      // get degree of all nodes in the spanning tree
      int[] degree = new int[cluster.size()];
      for(int i = 0; i < nodes.length; i++) {
        degree[nodes[i]]++;
      }
      // check if cluster contains any internal edges
      for(int i = 0, e = nodes.length - 1; i < e; i += 2) {
        if(degree[nodes[i]] > 1 && degree[nodes[i + 1]] > 1) {
          internalEdges[c] = true;
        }
      }

      clusterDegrees[c] = degree;

      // find maximum sparseness in the Minimum Spanning Tree
      for(int i = 0, e = nodes.length - 1; i < e; i += 2) {
        final int n1 = nodes[i], n2 = nodes[i + 1];
        // We only consider edges where both vertices are internal nodes.
        // If a cluster has no internal nodes we consider all edges.
        if(distances[n1][n2] > dscMax && (!internalEdges[c] || (degree[n1] > 1 && degree[n2] > 1))) {
          dscMax = distances[n1][n2];
        }
      }
      clusterDscMax[c] = dscMax;
    }

    // compute density separation of all clusters
    double dbcv = 0;
    for(int c = 0; c < numc; c++) {
      Cluster<?> cluster = clusters.get(c);
      if(cluster.isNoise() || cluster.size() < 2) {
        continue;
      }
      double currentDscMax = clusterDscMax[c];
      double[] clusterCoreDists = coreDists[c];
      int[] currentDegree = clusterDegrees[c];

      // minimal Density Separation of the Cluster
      double dspcMin = Double.POSITIVE_INFINITY;
      for(DBIDArrayIter it = cids[c].iter(); it.valid(); it.advance()) {
        // We again ignore external nodes, if the cluster has any internal
        // nodes.
        if(currentDegree[it.getOffset()] < 2 && internalEdges[c]) {
          continue;
        }
        double currentCoreDist = clusterCoreDists[it.getOffset()];
        for(int oc = 0; oc < numc; oc++) {
          Cluster<?> ocluster = clusters.get(oc);
          if(ocluster.isNoise() || ocluster.size() < 2 || cluster == ocluster) {
            continue;
          }
          int[] oDegree = clusterDegrees[oc];
          double[] oclusterCoreDists = coreDists[oc];
          for(DBIDArrayIter it2 = cids[oc].iter(); it2.valid(); it2.advance()) {
            if(oDegree[it2.getOffset()] < 2 && internalEdges[oc]) {
              continue;
            }
            double mutualReachDist = MathUtil.max(currentCoreDist, oclusterCoreDists[it2.getOffset()], dq.distance(it, it2));
            dspcMin = mutualReachDist < dspcMin ? mutualReachDist : dspcMin;
          }
        }
      }

      // compute DBCV
      double vc = (dspcMin - currentDscMax) / MathUtil.max(dspcMin, currentDscMax);
      double weight = cluster.size() / (double) rel.size();
      dbcv += weight * vc;
    }

    EvaluationResult ev = EvaluationResult.findOrCreate(db.getHierarchy(), cl, "Internal Clustering Evaluation", "internal evaluation");
    MeasurementGroup g = ev.findOrCreateGroup("Distance-based Evaluation");
    g.addMeasure("Density Based Clustering Validation", dbcv, 0., Double.POSITIVE_INFINITY, 0., true);
    db.getHierarchy().resultChanged(ev);
    return dbcv;
  }
109728
def build_relation_predicate(relations: Strings) -> EdgePredicate:
    """"""
    if isinstance(relations, str):
        @edge_predicate
        def relation_predicate(edge_data: EdgeData) -> bool:
            """Pass for relations matching the enclosed value."""
            return edge_data[RELATION] == relations

    elif isinstance(relations, Iterable):
        relation_set = set(relations)

        @edge_predicate
        def relation_predicate(edge_data: EdgeData) -> bool:
            """Pass for relations matching the enclosed values."""
            return edge_data[RELATION] in relation_set

    else:
        raise TypeError

    return relation_predicate
63416
public Iterator<DataPoint> getDataPointIterator()
    {
        Iterator<DataPoint> iteData = new Iterator<DataPoint>() 
        {
            int cur = 0;
            int to = size();

            @Override
            public boolean hasNext()
            {
                return cur < to;
            }

            @Override
            public DataPoint next()
            {
                return getDataPoint(cur++);
            }

            @Override
            public void remove()
            {
                throw new UnsupportedOperationException("This operation is not supported for DataSet");
            }
        };
        
        return iteData;
    }
818543
public List getPrefetchedRelationships()
	{
        // BRJ:
        // combine data from query and criteria
        // TODO: to be removed when Criteria#addPrefetchedRelationship is removed
        ArrayList temp = new ArrayList();
        temp.addAll(m_prefetchedRelationships);

        if (getCriteria() != null)
        {
            temp.addAll(getCriteria().getPrefetchedRelationships());
        }

        return temp;
    }