提取ngram
@param data
@param gramSizes
@return ngram字符串数组
338284
public static ArrayList<String> ngramOnCharacter2List(String data,int[] gramSizes) {
        // 提取ngram
        ArrayList<String> list = new ArrayList<String>();
        ngramOnCharacter(data, gramSizes, list);
        return list;
    }
845326
def NgramScorer(frequency_map):
    
    # Calculate the log probability
    length = len(next(iter(frequency_map)))
    # TODO: 0.01 is a magic number. Needs to be better than that.
    floor = math.log10(0.01 / sum(frequency_map.values()))
    ngrams = frequency.frequency_to_probability(frequency_map, decorator=math.log10)

    def inner(text):
        # I dont like this, it is only for the .upper() to work,
        # But I feel as though this can be removed in later refactoring
        text = ''.join(text)
        text = remove(text.upper(), string.whitespace + string.punctuation)
        return sum(ngrams.get(ngram, floor) for ngram in iterate_ngrams(text, length))

    return inner
685376
function grams(value) {
    var nGrams = []
    var index

    if (value === null || value === undefined) {
      return nGrams
    }

    value = value.slice ? value : String(value)
    index = value.length - n + 1

    if (index < 1) {
      return nGrams
    }

    while (index--) {
      nGrams[index] = value.slice(index, index + n)
    }

    return nGrams
  }
163510
def extract_ngram_filter(pos_seq, regex=SimpleNP, minlen=1, maxlen=8):
	""""""
	ss = coarse_tag_str(pos_seq)

	def gen():
		for s in xrange(len(ss)):
			for n in xrange(minlen, 1 + min(maxlen, len(ss) - s)):
				e = s + n
				substr = ss[s:e]
				if re.match(regex + "$", substr):
					yield (s, e)

	return list(gen())
298158
private HashMap<String, Object> getAggregationMap(JsonReader in) throws IOException {
    HashMap<String, Object> objMap = new HashMap<>();
    while (in.peek() != JsonToken.END_DOCUMENT) {
      parseNext(in, objMap);
    }
    return objMap;
  }
1103660
def ngrams(self, n=3):
    
    if n <= 0:
        return []
    grams = [WordList(self.words[i:i+n], parent=self)
                        for i in range(len(self.words) - n + 1)]
    return grams
374985
function nGram(n) {
  if (typeof n !== 'number' || isNaN(n) || n < 1 || n === Infinity) {
    throw new Error('`' + n + '` is not a valid argument for n-gram')
  }

  return grams

  // Create n-grams from a given value.
  function grams(value) {
    var nGrams = []
    var index

    if (value === null || value === undefined) {
      return nGrams
    }

    value = value.slice ? value : String(value)
    index = value.length - n + 1

    if (index < 1) {
      return nGrams
    }

    while (index--) {
      nGrams[index] = value.slice(index, index + n)
    }

    return nGrams
  }
}
745510
public boolean contains(String[] words) {
    if (words.length == 1) {
      return wordMap.contains(words[0]);
    }
    else {
      NGram ngram = nGramFactory.createNGram(words);
      if (ngram == null) {
        return false;
      }
      else {
        return gramSet.contains(ngram);
      }
    }
  }
128220
public NGram createNGram(List words) {
    int[] nums = new int[words.size()]; //ngram needs it own copy of this array so keep making a unique one for each instance.
    for (int wi=0;wi<nums.length;wi++) {
      nums[wi] = wordMap.getIndex(words.get(wi));
      if (nums[wi] == -1) {
        return null;
      }
    }
    return new NGram(nums);
  }
141135
def get_filter_ngrams_pattern(filter_ngrams):
        
        return re.compile('|'.join([re.escape(ngram) for ngram in
                                    filter_ngrams]))
994039
protected void loadGrams(DataInputStream input) throws IOException {
    gramSet = new HashSet();
    try {
      while(true) {
        int gramLength=input.readInt();
      
        int[] words = new int[gramLength];
        for (int wi=0;wi<gramLength;wi++) {
          words[wi]=input.readInt();
        }
        gramSet.add(new NGram(words));
      }
    }
    catch(EOFException e) {
      
    }
  }
1190366
function(data) {
      var bytesToPad, padding;
      bytesToPad = BLOCKSIZE - (data.length % BLOCKSIZE);
      padding = this.randomBytes(bytesToPad);
      return this.concat([padding, data]);
    }
33450
public FgExampleList getData(LogLinearXYData data) {
        IntObjectBimap<String> alphabet = data.getFeatAlphabet();
        List<LogLinearExample> exList = data.getData();    
        if (this.alphabet == null) {
            this.alphabet = alphabet;
            this.stateNames = getStateNames(exList, data.getYAlphabet());
        }
        
        if (prm.cacheExamples) {
            FgExampleMemoryStore store = new FgExampleMemoryStore();
            for (final LogLinearExample desc : exList) {
                LFgExample ex = getFgExample(desc);
                store.add(ex);
            }
            return store;
        } else {
            return new FgExampleList() {
                @Override
                public LFgExample get(int i) {
                    return getFgExample(exList.get(i));
                }
                @Override
                public int size() {
                    return exList.size();
                }                
            };
        }
    }
258540
def create_ngram_set(input_list, ngram_value=2):
    
    return set(zip(*[input_list[i:] for i in range(ngram_value)]))
522820
protected WsByteBuffer[] putInt(int data, WsByteBuffer[] buffers) {
        return putBytes(GenericUtils.asBytes(data), buffers);
    }
592249
def _generate_ngram_table(self, output_dir, labels, results):
        """"""
        html = []
        grouped = results.groupby(constants.NGRAM_FIELDNAME)
        row_template = self._generate_ngram_row_template(labels)
        for name, group in grouped:
            html.append(self._render_ngram_row(name, group, row_template,
                                               labels))
        return '\n'.join(html)
666429
@NotNull
    public Map<String,Integer> extractCountedGrams(@NotNull CharSequence text) {
        text = applyPadding(text);
        int len = text.length();

        int initialCapacity = 0;
        for (Integer gramLength : gramLengths) {
            initialCapacity += guessNumDistinctiveGrams(len, gramLength);
        }

        Map<String,Integer> grams = new LinkedHashMap<>(initialCapacity);
        for (Integer gramLength : gramLengths) {
            _extractCounted(text, gramLength, len, grams);
        }
        return grams;
    }