命名实体识别

@param wordArray
@param posArray
@return
490565
public String[] namedEntityRecognize(String[] wordArray, String[] posArray)
    {
        if (neRecognizer == null)
        {
            throw new IllegalStateException("未提供命名实体识别模型");
        }
        return recognize(wordArray, posArray);
    }
596331
public String tag(final String word, final String postag) {
    // lookup postag as value of the map
    String newPosTag = this.dictMap.get(word.toLowerCase());
    if (newPosTag == null) {
      newPosTag = postag;
    }
    return newPosTag;
  }
960395
public static String[] toTypeNameArray(Class<?>[] c) {
        if (c == null)
            return null;
        String[] s = new String[c.length];
        for (int i = 0; i < c.length; i++) {
            s[i] = c[i].getName();
        }
        return s;
    }
58142
public JSONArray names() {
        return nameValuePairs.isEmpty()
                ? null
                : new JSONArray(new ArrayList<String>(nameValuePairs.keySet()));
    }
58193
protected String getAlgorithm(JSONArray customInfoArray) {
        StringBuffer sb = new StringBuffer();
        for (int i = 0; i < customInfoArray.size(); i++) {
            JSONObject customInfo = (JSONObject) customInfoArray.get(i);
            sb.append("|").append(customInfo.get("name"));
        }
        return sb.toString();
    }
1106818
protected static void generateWord(List<Vertex> linkedArray, WordNet wordNetOptimum)
    {
        fixResultByRule(linkedArray);

        //--------------------------------------------------------------------
        // 建造新词网
        wordNetOptimum.addAll(linkedArray);
    }
1169337
public String[] getTags(String word) {
    if (caseSensitive) {
      return (String[]) dictionary.get(word);
    }
    else {
      //System.err.println(java.util.Arrays.asList((String[]) dictionary.get(word.toLowerCase())));
      return (String[]) dictionary.get(word.toLowerCase());
    }
  }
55696
public String[][] tag2DoubleArray(String src) {
		if(src==null||src.length()==0)
			return null;
		String[] sents = Sentenizer.split(src);
		String[][] words = new String[sents.length][];
		for(int i=0;i<sents.length;i++){
			words[i] = tag2Array(sents[i]);
		}		
		return words;
	}
1326613
public static Map<String, String> paraFilter(Map<String, String> sArray) {
		Map<String, String> result = new HashMap<String, String>();
		if (sArray == null || sArray.size() <= 0) {
			return result;
		}
		for (String key : sArray.keySet()) {
			String value = sArray.get(key);
			if (value == null || value.equals("") || key.equalsIgnoreCase("sign")
					|| key.equalsIgnoreCase("sign_type")) {
				continue;
			}
			result.put(key, value);
		}
		return result;
	}
411288
protected int[] extractFeature(String[] wordArray, String[] posArray, FeatureMap featureMap, int position)
    {
        List<Integer> featVec = new ArrayList<Integer>();

        String pre2Word = position >= 2 ? wordArray[position - 2] : "_B_";
        String preWord = position >= 1 ? wordArray[position - 1] : "_B_";
        String curWord = wordArray[position];
        String nextWord = position <= wordArray.length - 2 ? wordArray[position + 1] : "_E_";
        String next2Word = position <= wordArray.length - 3 ? wordArray[position + 2] : "_E_";

        String pre2Pos = position >= 2 ? posArray[position - 2] : "_B_";
        String prePos = position >= 1 ? posArray[position - 1] : "_B_";
        String curPos = posArray[position];
        String nextPos = position <= posArray.length - 2 ? posArray[position + 1] : "_E_";
        String next2Pos = position <= posArray.length - 3 ? posArray[position + 2] : "_E_";

        StringBuilder sb = new StringBuilder();
        addFeatureThenClear(sb.append(pre2Word).append('1'), featVec, featureMap);
        addFeatureThenClear(sb.append(preWord).append('2'), featVec, featureMap);
        addFeatureThenClear(sb.append(curWord).append('3'), featVec, featureMap);
        addFeatureThenClear(sb.append(nextWord).append('4'), featVec, featureMap);
        addFeatureThenClear(sb.append(next2Word).append('5'), featVec, featureMap);
//        addFeatureThenClear(sb.append(pre2Word).append(preWord).append('6'), featVec, featureMap);
//        addFeatureThenClear(sb.append(preWord).append(curWord).append('7'), featVec, featureMap);
//        addFeatureThenClear(sb.append(curWord).append(nextWord).append('8'), featVec, featureMap);
//        addFeatureThenClear(sb.append(nextWord).append(next2Word).append('9'), featVec, featureMap);

        addFeatureThenClear(sb.append(pre2Pos).append('A'), featVec, featureMap);
        addFeatureThenClear(sb.append(prePos).append('B'), featVec, featureMap);
        addFeatureThenClear(sb.append(curPos).append('C'), featVec, featureMap);
        addFeatureThenClear(sb.append(nextPos).append('D'), featVec, featureMap);
        addFeatureThenClear(sb.append(next2Pos).append('E'), featVec, featureMap);
        addFeatureThenClear(sb.append(pre2Pos).append(prePos).append('F'), featVec, featureMap);
        addFeatureThenClear(sb.append(prePos).append(curPos).append('G'), featVec, featureMap);
        addFeatureThenClear(sb.append(curPos).append(nextPos).append('H'), featVec, featureMap);
        addFeatureThenClear(sb.append(nextPos).append(next2Pos).append('I'), featVec, featureMap);

        return toFeatureArray(featVec);
    }
464876
private static void combineWords(String[] wordNet, int start, int end, CoreDictionary.Attribute[] attributeArray, CoreDictionary.Attribute value)
    {
        if (start + 1 != end)   // 小优化，如果只有一个词，那就不需要合并，直接应用新属性
        {
            StringBuilder sbTerm = new StringBuilder();
            for (int j = start; j < end; ++j)
            {
                if (wordNet[j] == null) continue;
                sbTerm.append(wordNet[j]);
                wordNet[j] = null;
            }
            wordNet[start] = sbTerm.toString();
        }
        attributeArray[start] = value;
    }
816552
public double[] getWordVector(String word) {
        INDArray r = getWordVectorMatrix(word);
        if (r == null)
            return null;
        return r.dup().data().asDouble();
    }
241457
@Override
    public INDArray getWordVectorMatrix(String word) {
        // TODO: add variable UNK here
        int idx = 0;
        if (hasWord(word))
            idx = vocabCache.indexOf(word);
        else if (getUNK() != null)
            idx = vocabCache.indexOf(getUNK());
        else
            return null;

        int deviceId = Nd4j.getAffinityManager().getDeviceForCurrentThread();
        INDArray array = null;

        if (cachePerDevice > 0 && cacheWrtDevice.get(deviceId).containsKey(idx))
            return cacheWrtDevice.get(Nd4j.getAffinityManager().getDeviceForCurrentThread()).get(idx);

        array = storage.get(idx);

        if (cachePerDevice > 0) {
            // TODO: add cache here
            long arrayBytes = array.length() * array.data().getElementSize();
            if ((arrayBytes * cacheWrtDevice.get(deviceId).size()) + arrayBytes < cachePerDevice)
                cacheWrtDevice.get(deviceId).put(idx, array);
        }

        return array;
    }
973578
private boolean isPart(PartOfSpeech pos, String word){
		if(NP.contains(pos))
			return true;
		
		if(pos == PartOfSpeech.结构助词 && word.equals("的"))
			return true;
		
		return false;
	}
612194
public StructuredQueryDefinition word(TextIndex index, String... words) {
    return new WordQuery(index, null, null, null, words);
  }
630886
public WordInfo getNextN(String[] data, int index, int N) {
		StringBuilder sb = new StringBuilder();
		int i = index;
		while(sb.length()<N&&i<data.length){			
			sb.append(data[i]);
			i++;
		}
		if(sb.length()<=N)	
			return new WordInfo(sb.toString(),i-index);
		else
			return new WordInfo(sb.substring(0,N),i-index);
	}
49350
@Nullable
  public AnalyzedTokenReadings lookup(String word) throws IOException {
    List<AnalyzedTokenReadings> result = tag(Collections.singletonList(word), false);
    AnalyzedTokenReadings atr = result.get(0);
    if (atr.getAnalyzedToken(0).getPOSTag() == null) {
      return null;
    }
    return atr;
  }