使用用户词典合并粗分结果，并将用户词语收集到全词图中
@param vertexList 粗分结果
@param wordNetAll 收集用户词语到全词图中
@return 合并后的结果
976845
protected static List<Vertex> combineByCustomDictionary(List<Vertex> vertexList, final WordNet wordNetAll)
    {
        return combineByCustomDictionary(vertexList, CustomDictionary.dat, wordNetAll);
    }
1083093
public VertexList vertexIds() {
        LongArrayList list = new LongArrayList();
        long previousId = 0;
        for (Long id : Iterables.transform(this,new Function<Entry, Long>() {
            @Nullable
            @Override
            public Long apply(@Nullable Entry entry) {
                return edgeSerializer.readRelation(entry,true,tx).getOtherVertexId();
            }
        })) {
            list.add(id);
            if (id>=previousId && previousId>=0) previousId=id;
            else previousId=-1;
        }
        return new VertexLongList(tx,list,previousId>=0);
    }
1097690
private static void createTermvectorFull(
      List<ComponentTermVector> termVectorList,
      Map<Integer, Integer> positionsData, List<Integer> docSet, Terms t,
      LeafReader r, LeafReaderContext lrc) throws IOException {
    if (t != null) {
      BytesRef term;
      TermsEnum termsEnum;
      PostingsEnum postingsEnum = null;
      String segmentName = "segment" + lrc.ord;
      int segmentNumber = lrc.parent.leaves().size();
      // loop over termvectors
      for (ComponentTermVector termVector : termVectorList) {
        if (termVector.full || termVector.list != null) {
          if (termVector.full) {
            termVector.subComponentFunction.dataCollector.setWithTotal();
          }
          List<CompiledAutomaton> listAutomata;
          Map<String, Automaton> automatonMap;
          if (termVector.list == null) {
            listAutomata = new ArrayList<>();
            CompiledAutomaton compiledAutomaton;
            Automaton automaton;
            if ((termVector.regexp == null) || (termVector.regexp.isEmpty())) {
              RegExp re = new RegExp(
                  termVector.prefix + MtasToken.DELIMITER + ".*");
              automaton = re.toAutomaton();
            } else {
              RegExp re = new RegExp(termVector.prefix + MtasToken.DELIMITER
                  + termVector.regexp + "\u0000*");
              automaton = re.toAutomaton();
            }
            compiledAutomaton = new CompiledAutomaton(automaton);
            listAutomata.add(compiledAutomaton);
          } else {
            automatonMap = MtasToken.createAutomatonMap(termVector.prefix,
                new ArrayList<String>(termVector.list),
                termVector.listRegexp ? false : true);
            listAutomata = MtasToken.createAutomata(termVector.prefix,
                termVector.regexp, automatonMap);
          }
          List<ByteRunAutomaton> ignoreByteRunAutomatonList = null;
          if ((termVector.ignoreRegexp != null)
              && (!termVector.ignoreRegexp.isEmpty())) {
            ignoreByteRunAutomatonList = new ArrayList<>();
            RegExp re = new RegExp(termVector.prefix + MtasToken.DELIMITER
                + termVector.ignoreRegexp + "\u0000*");
            ignoreByteRunAutomatonList
                .add(new ByteRunAutomaton(re.toAutomaton()));
          }
          if (termVector.ignoreList != null) {
            if (ignoreByteRunAutomatonList == null) {
              ignoreByteRunAutomatonList = new ArrayList<>();
            }
            Map<String, Automaton> list = MtasToken.createAutomatonMap(
                termVector.prefix, new ArrayList<String>(termVector.ignoreList),
                termVector.ignoreListRegexp ? false : true);
            for (Automaton automaton : list.values()) {
              ignoreByteRunAutomatonList.add(new ByteRunAutomaton(automaton));
            }
          }
          for (CompiledAutomaton compiledAutomaton : listAutomata) {
            if (!compiledAutomaton.type
                .equals(CompiledAutomaton.AUTOMATON_TYPE.NORMAL)) {
              if (compiledAutomaton.type
                  .equals(CompiledAutomaton.AUTOMATON_TYPE.NONE)) {
                // do nothing
              } else {
                throw new IOException(
                    "compiledAutomaton is " + compiledAutomaton.type);
              }
            } else {
              termsEnum = t.intersect(compiledAutomaton, null);
              int initSize = Math.min((int) t.size(), 1000);
              termVector.subComponentFunction.dataCollector.initNewList(
                  initSize, segmentName, segmentNumber, termVector.boundary);
              boolean doBasic = termVector.subComponentFunction.dataCollector
                  .getStatsType().equals(CodecUtil.STATS_BASIC);
              if (termVector.functions != null) {
                for (SubComponentFunction function : termVector.functions) {
                  function.dataCollector.initNewList(initSize);
                  doBasic = doBasic
                      ? (function.parserFunction.sumRule()
                          && !function.parserFunction.needPositions()
                          && function.dataCollector.getStatsType()
                              .equals(CodecUtil.STATS_BASIC))
                      : doBasic;
                }
              }
              // only if documents
              if (!docSet.isEmpty()) {
                int termDocId;
                boolean acceptedTerm;
                String key;
                // loop over terms
                while ((term = termsEnum.next()) != null) {
                  if (validateTermWithStartValue(term, termVector)
                      && validateTermWithDistance(term, termVector)) {
                    termDocId = -1;
                    acceptedTerm = true;
                    if (ignoreByteRunAutomatonList != null) {
                      for (ByteRunAutomaton ignoreByteRunAutomaton : ignoreByteRunAutomatonList) {
                        if (ignoreByteRunAutomaton.run(term.bytes, term.offset,
                            term.length)) {
                          acceptedTerm = false;
                          break;
                        }
                      }
                    }
                    if (acceptedTerm) {
                      if (doBasic) {
                        // compute numbers;
                        TermvectorNumberBasic numberBasic = computeTermvectorNumberBasic(
                            docSet, termDocId, termsEnum, r, lrc, postingsEnum);
                        // register
                        if (numberBasic.docNumber > 0) {
                          long valueLong = 0;
                          key = MtasToken.getPostfixFromValue(term);
                          try {
                            valueLong = termVector.subComponentFunction.parserFunction
                                .getValueLong(numberBasic.valueSum, 1);
                          } catch (IOException e) {
                            log.debug(e);
                            termVector.subComponentFunction.dataCollector.error(
                                MtasToken.getPostfixFromValue(term),
                                e.getMessage());
                          }
                          termVector.subComponentFunction.dataCollector.add(key,
                              valueLong, numberBasic.docNumber);
                          if (termVector.functions != null) {
                            for (SubComponentFunction function : termVector.functions) {
                              if (function.dataType
                                  .equals(CodecUtil.DATA_TYPE_LONG)) {
                                long valueFunction = function.parserFunction
                                    .getValueLong(numberBasic.valueSum, 0);
                                function.dataCollector.add(key, valueFunction,
                                    numberBasic.docNumber);
                              } else if (function.dataType
                                  .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
                                double valueFunction = function.parserFunction
                                    .getValueDouble(numberBasic.valueSum, 0);
                                function.dataCollector.add(key, valueFunction,
                                    numberBasic.docNumber);
                              }
                            }
                          }
                        }
                      } else {
                        TermvectorNumberFull numberFull = computeTermvectorNumberFull(
                            docSet, termDocId, termsEnum, lrc, postingsEnum,
                            positionsData);
                        if (numberFull.docNumber > 0) {
                          long[] valuesLong = new long[numberFull.docNumber];
                          key = MtasToken.getPostfixFromValue(term);
                          for (int i = 0; i < numberFull.docNumber; i++) {
                            try {
                              valuesLong[i] = termVector.subComponentFunction.parserFunction
                                  .getValueLong(
                                      new long[] { numberFull.args[i] },
                                      numberFull.positions[i]);
                            } catch (IOException e) {
                              log.debug(e);
                              termVector.subComponentFunction.dataCollector
                                  .error(key, e.getMessage());
                            }
                          }
                          termVector.subComponentFunction.dataCollector.add(key,
                              valuesLong, valuesLong.length);
                          if (termVector.functions != null) {
                            for (SubComponentFunction function : termVector.functions) {
                              if (function.dataType
                                  .equals(CodecUtil.DATA_TYPE_LONG)) {
                                valuesLong = new long[numberFull.docNumber];
                                for (int i = 0; i < numberFull.docNumber; i++) {
                                  try {
                                    valuesLong[i] = function.parserFunction
                                        .getValueLong(
                                            new long[] { numberFull.args[i] },
                                            numberFull.positions[i]);
                                  } catch (IOException e) {
                                    log.debug(e);
                                    function.dataCollector.error(key,
                                        e.getMessage());
                                  }
                                }
                                function.dataCollector.add(key, valuesLong,
                                    valuesLong.length);
                              } else if (function.dataType
                                  .equals(CodecUtil.DATA_TYPE_DOUBLE)) {
                                double[] valuesDouble = new double[numberFull.docNumber];
                                for (int i = 0; i < numberFull.docNumber; i++) {
                                  try {
                                    valuesDouble[i] = function.parserFunction
                                        .getValueDouble(
                                            new long[] { numberFull.args[i] },
                                            numberFull.positions[i]);
                                  } catch (IOException e) {
                                    log.debug(e);
                                    function.dataCollector.error(key,
                                        e.getMessage());
                                  }
                                }
                                function.dataCollector.add(key, valuesDouble,
                                    valuesDouble.length);
                              }
                            }
                          }
                        }

                      }
                    }
                  }
                }
              }
              termVector.subComponentFunction.dataCollector.closeNewList();
              if (termVector.functions != null) {
                for (SubComponentFunction function : termVector.functions) {
                  function.dataCollector.closeNewList();
                }
              }
            }
          }
        }
      }
    }
  }
97401
public AbstractExpression getAllFilters() {
        ArrayDeque<JoinNode> joinNodes = new ArrayDeque<>();
        ArrayDeque<AbstractExpression> in = new ArrayDeque<>();
        ArrayDeque<AbstractExpression> out = new ArrayDeque<>();
        // Iterate over the join nodes to collect their join and where expressions
        joinNodes.add(this);
        while (!joinNodes.isEmpty()) {
            JoinNode joinNode = joinNodes.poll();
            if (joinNode.m_joinExpr != null) {
                in.add(joinNode.m_joinExpr);
            }
            if (joinNode.m_whereExpr != null) {
                in.add(joinNode.m_whereExpr);
            }
            joinNode.queueChildren(joinNodes);
        }

        // this chunk of code breaks the code into a list of expression that
        // all have to be true for the where clause to be true
        AbstractExpression inExpr = null;
        while ((inExpr = in.poll()) != null) {
            if (inExpr.getExpressionType() == ExpressionType.CONJUNCTION_AND) {
                in.add(inExpr.getLeft());
                in.add(inExpr.getRight());
            }
            else {
                out.add(inExpr);
            }
        }
        return ExpressionUtil.combinePredicates(out);
    }
1280142
private List<IndexTerm> cloneIndextermList (final List<IndexTerm> termList){
        final List<IndexTerm> termListClone = new ArrayList<>(termList.size());
        if (!termList.isEmpty()){
            termListClone.addAll(termList);
        }
        return termListClone;
    }
1049557
public void insert(int line, Vertex vertex, WordNet wordNetAll)
    {
        for (Vertex oldVertex : vertexes[line])
        {
            // 保证唯一性
            if (oldVertex.realWord.length() == vertex.realWord.length()) return;
        }
        vertexes[line].add(vertex);
        ++size;
        // 保证这个词语前面直连
        final int start = Math.max(0, line - 5); // 效率起见，只扫描前4行
        for (int l = line - 1; l > start; --l)
        {
            LinkedList<Vertex> all = wordNetAll.get(l);
            if (all.size() <= vertexes[l].size())
                continue;
            for (Vertex pre : all)
            {
                if (pre.length() + l == line)
                {
                    vertexes[l].add(pre);
                    ++size;
                }
            }
        }
        // 保证这个词语后面直连
        int l = line + vertex.realWord.length();
        LinkedList<Vertex> targetLine = wordNetAll.get(l);
        if (vertexes[l].size() == 0 && targetLine.size() != 0) // 有时候vertexes里面的词语已经经过用户词典合并，造成数量更少
        {
            size += targetLine.size();
            vertexes[l] = targetLine;
        }
    }
637727
private List<String> getAllTokens(SStructuredNode n, String name, TComponenttype c, long cnr, String namespace)
  {
    List<String> result = null;
    if (!visitedNodes.contains(n.getId()))
    {
      result = new LinkedList<String>();
      visitedNodes.add(n.getId());
      if (tokensOfNode.containsKey(n.getId()))
      {
        for (String t : tokensOfNode.get(n.getId()))
        {
          result.add(t);
          if (componentOfToken.get(t) == null)
          {
            List<Long> newlist = new LinkedList<Long>();
            newlist.add(cnr);
            componentOfToken.put(t, newlist);
          }
          else
          {
            if (!componentOfToken.get(t).contains(cnr))
            {
              componentOfToken.get(t).add(cnr);
            }
          }
        }
      }
      else
      {
        result = searchTokens(n, cnr);
        if (result != null)
        {
          tokensOfNode.put(n.getId(), result);
        }
      }
      //get "P"-Edges!
      List<SRelation<SNode,SNode>> outEdges = n.getGraph().getOutRelations(n.getId());
      if(outEdges != null)
      {
        for (SRelation<? extends SNode,? extends SNode> e : outEdges)
        {
          if(includeEdge(e, namespace))
          {
            SPointingRelation rel = (SPointingRelation) e;
            if (name.equals(componentNameForRelation(rel))
              && !visitedNodes.contains(rel.getTarget().getId()))
            {
              c.nodeList.add(rel.getTarget().getId());
              List<String> Med = getAllTokens(rel.getTarget(), 
                name, c, cnr, namespace);
              for (String l : Med)
              {
                if (result != null && !result.contains(l))
                {
                  result.add(l);
                }
              }
            }
          }
        }
      }
      List<SRelation<SNode,SNode>> inEdges = n.getGraph().getInRelations(n.getId());
      if(inEdges != null)
      {
        for (SRelation<? extends SNode,? extends SNode> e : inEdges)
        {
          if(includeEdge(e, namespace))
          {
            SPointingRelation rel = (SPointingRelation) e;
            if (name.equals(componentNameForRelation(rel))
              && !visitedNodes.contains(rel.getSource().getId()))
            {
              c.nodeList.add(rel.getSource().getId());
              List<String> Med = getAllTokens(rel.getSource(), name, c, cnr, namespace);
              for (String s : Med)
              {
                if (result != null && !result.contains(s))
                {
                  result.add(s);
                }
              }
            }
          }
        }
      }
    }
    return result;
  }
624403
public List<ManyToOne<EmbeddableAttributes<T>>> getAllManyToOne()
   {
      List<ManyToOne<EmbeddableAttributes<T>>> list = new ArrayList<ManyToOne<EmbeddableAttributes<T>>>();
      List<Node> nodeList = childNode.get("many-to-one");
      for(Node node: nodeList)
      {
         ManyToOne<EmbeddableAttributes<T>>  type = new ManyToOneImpl<EmbeddableAttributes<T>>(this, "many-to-one", childNode, node);
         list.add(type);
      }
      return list;
   }
206955
public void addAll(List<Vertex> vertexList)
    {
        int i = 0;
        for (Vertex vertex : vertexList)
        {
            add(i, vertex);
            i += vertex.realWord.length();
        }
    }
40741
protected static List<Vertex> combineByCustomDictionary(List<Vertex> vertexList, DoubleArrayTrie<CoreDictionary.Attribute> dat)
    {
        assert vertexList.size() >= 2 : "vertexList至少包含 始##始 和 末##末";
        Vertex[] wordNet = new Vertex[vertexList.size()];
        vertexList.toArray(wordNet);
        // DAT合并
        int length = wordNet.length - 1; // 跳过首尾
        for (int i = 1; i < length; ++i)
        {
            int state = 1;
            state = dat.transition(wordNet[i].realWord, state);
            if (state > 0)
            {
                int to = i + 1;
                int end = to;
                CoreDictionary.Attribute value = dat.output(state);
                for (; to < length; ++to)
                {
                    state = dat.transition(wordNet[to].realWord, state);
                    if (state < 0) break;
                    CoreDictionary.Attribute output = dat.output(state);
                    if (output != null)
                    {
                        value = output;
                        end = to + 1;
                    }
                }
                if (value != null)
                {
                    combineWords(wordNet, i, end, value);
                    i = end - 1;
                }
            }
        }
        // BinTrie合并
        if (CustomDictionary.trie != null)
        {
            for (int i = 1; i < length; ++i)
            {
                if (wordNet[i] == null) continue;
                BaseNode<CoreDictionary.Attribute> state = CustomDictionary.trie.transition(wordNet[i].realWord.toCharArray(), 0);
                if (state != null)
                {
                    int to = i + 1;
                    int end = to;
                    CoreDictionary.Attribute value = state.getValue();
                    for (; to < length; ++to)
                    {
                        if (wordNet[to] == null) continue;
                        state = state.transition(wordNet[to].realWord.toCharArray(), 0);
                        if (state == null) break;
                        if (state.getValue() != null)
                        {
                            value = state.getValue();
                            end = to + 1;
                        }
                    }
                    if (value != null)
                    {
                        combineWords(wordNet, i, end, value);
                        i = end - 1;
                    }
                }
            }
        }
        vertexList.clear();
        for (Vertex vertex : wordNet)
        {
            if (vertex != null) vertexList.add(vertex);
        }
        return vertexList;
    }
649984
public List vertexList() {
        List vertices = new ArrayList(edgeSet().size());

        Object startVertex = vertexSet().iterator().next();

        Object vertex = startVertex;
        Object previousVertex = null;
        Object nextVertex = null;

        while (nextVertex != startVertex) {
            assert (degreeOf(vertex) == 2);
            List edges = edgesOf(vertex);

            vertices.add(vertex);

            Edge edge = (Edge) edges.get(0);
            nextVertex = edge.oppositeVertex(vertex);

            if (nextVertex == previousVertex) {
                edge = (Edge) edges.get(1);
                nextVertex = edge.oppositeVertex(vertex);
            }

            previousVertex = vertex;
            vertex = nextVertex;

        }

        return vertices;
    }
753571
public static final String vertexAsString(Vertex vertex, int depth, String withEdgesOfLabel)
    {
        StringBuilder sb = new StringBuilder();
        vertexAsString(vertex, depth, withEdgesOfLabel, sb, 0, new HashSet<>());
        return sb.toString();
    }
896332
public List<GeoObject> getMentionedGnObjects(List<String> tokenList){

		//logger.info("tokenList: " + tokenList);
		Set<Integer> gnIds = new HashSet<Integer>();
		
		for (int i = 0; i < tokenList.size(); i++){
			Set<Integer> ids = gnObjectMapNameLookup.get(tokenList.get(i));
			if (ids != null){
				gnIds.addAll(ids);
			}
			//logger.info("t: " + tokenList.get(i) + " -> " + ids);
		}
		// use also pairs of consecutive tokens
		if (tokenList.size() > 1) {
			for (int i = 0; i < tokenList.size() - 1; i++){
				Set<Integer> ids = gnObjectMapNameLookup.get(tokenList.get(i) + " " + tokenList.get(i+1));
				if (ids != null){
					gnIds.addAll(ids);
				}
				//logger.info("t: " + tokenList.get(i) + " " + tokenList.get(i+1) + " -> " + ids);
			}
		}
		
		List<GeoObject> locations = new ArrayList<GeoObject>(gnIds.size());
		for (Integer id : gnIds) {
			locations.add(gnObjectMap.get(id));
		}
		return locations;
	}
386738
public static void parsePattern(List<NS> nsList, List<Vertex> vertexList, final WordNet wordNetOptimum, final WordNet wordNetAll)
    {
//        ListIterator<Vertex> listIterator = vertexList.listIterator();
        StringBuilder sbPattern = new StringBuilder(nsList.size());
        for (NS ns : nsList)
        {
            sbPattern.append(ns.toString());
        }
        String pattern = sbPattern.toString();
        final Vertex[] wordArray = vertexList.toArray(new Vertex[0]);
        trie.parseText(pattern, new AhoCorasickDoubleArrayTrie.IHit<String>()
        {
            @Override
            public void hit(int begin, int end, String value)
            {
                StringBuilder sbName = new StringBuilder();
                for (int i = begin; i < end; ++i)
                {
                    sbName.append(wordArray[i].realWord);
                }
                String name = sbName.toString();
                // 对一些bad case做出调整
                if (isBadCase(name)) return;

                // 正式算它是一个名字
                if (HanLP.Config.DEBUG)
                {
                    System.out.printf("识别出地名：%s %s\n", name, value);
                }
                int offset = 0;
                for (int i = 0; i < begin; ++i)
                {
                    offset += wordArray[i].realWord.length();
                }
                wordNetOptimum.insert(offset, new Vertex(Predefine.TAG_PLACE, name, ATTRIBUTE, WORD_ID), wordNetAll);
            }
        });
    }
165600
public static Map<String, Long> buildCountMap(List<String> wordList) {
        return buildCountMap(wordList.stream());
    }
359395
public String[] partOfSpeechTag(List<String> wordList)
    {
        if (posTagger == null)
        {
            throw new IllegalStateException("未提供词性标注模型");
        }
        return tag(wordList);
    }
3171
public List<List<Word>> getSimpleSentenceList(Set<String> labelSet)
    {
        List<List<Word>> simpleList = new LinkedList<List<Word>>();
        for (Sentence sentence : sentenceList)
        {
            List<Word> wordList = new LinkedList<Word>();
            for (IWord word : sentence.wordList)
            {
                if (word instanceof CompoundWord)
                {
                    if (labelSet.contains(word.getLabel()))
                    {
                        for (Word inner : ((CompoundWord) word).innerList)
                        {
                            wordList.add(inner);
                        }
                    }
                    else
                    {
                        wordList.add(((CompoundWord) word).toWord());
                    }
                }
                else
                {
                    wordList.add((Word) word);
                }
            }
            simpleList.add(wordList);
        }

        return simpleList;
    }