Parameters
		----------
		ngram str or unicode, string to search for

		Returns
		-------
		pd.DataFrame, {'texts': <matching texts>, 'categories': <corresponding categories>}
324125
def search(self, ngram):
		'''
		

		'''
		mask = self._document_index_mask(ngram)
		return pd.DataFrame({
			'text': self.get_texts()[mask],
			'category': [self._category_idx_store.getval(idx)
			             for idx in self._y[mask]]
		})
1213446
def insert_ngram(self, ngram, count):
        
        query = "INSERT INTO _{0}_gram {1};".format(len(ngram),
            self._build_values_clause(ngram, count))
        self.execute_sql(query)
976616
def _load_ngram(name):
    
    module = importlib.import_module('lantern.analysis.english_ngrams.{}'.format(name))
    return getattr(module, name)
1366343
def _get_word_ngrams(n, sentences):
    
    assert len(sentences) > 0
    assert n > 0

    words = _split_into_words(sentences)
    return _get_ngrams(n, words)
838118
def get_ngram_counts(self):
        ''' 
        '''
        ngram_counts = {
            'classes': [],
            'corpus': 0
        }
        doc_ids = self.term_count_n.keys()
        doc_ids.sort()
        for doc_id in doc_ids:
            print self.term_count_n[doc_id]
            class_ngrams = len(self.term_count_n[doc_id]['ngrams'])
            ngram_counts['classes'].append(class_ngrams)
        corpus_ngrams = len(self.corpus_count_n['ngrams'])
        ngram_counts['corpus'] = corpus_ngrams
        return ngram_counts
1103660
def ngrams(self, n=3):
    
    if n <= 0:
        return []
    grams = [WordList(self.words[i:i+n], parent=self)
                        for i in range(len(self.words) - n + 1)]
    return grams
625682
def get_strings_from_utterance(tokenized_utterance: List[Token]) -> Dict[str, List[int]]:
    
    string_linking_scores: Dict[str, List[int]] = defaultdict(list)

    for index, token in enumerate(tokenized_utterance):
        for string in ATIS_TRIGGER_DICT.get(token.text.lower(), []):
            string_linking_scores[string].append(index)

    token_bigrams = bigrams([token.text for token in tokenized_utterance])
    for index, token_bigram in enumerate(token_bigrams):
        for string in ATIS_TRIGGER_DICT.get(' '.join(token_bigram).lower(), []):
            string_linking_scores[string].extend([index,
                                                  index + 1])

    trigrams = ngrams([token.text for token in tokenized_utterance], 3)
    for index, trigram in enumerate(trigrams):
        if trigram[0] == 'st':
            natural_language_key = f'st. {trigram[2]}'.lower()
        else:
            natural_language_key = ' '.join(trigram).lower()
        for string in ATIS_TRIGGER_DICT.get(natural_language_key, []):
            string_linking_scores[string].extend([index,
                                                  index + 1,
                                                  index + 2])
    return string_linking_scores
742387
def _generate_ngrams_with_context_helper(ngrams_iter: iter, ngrams_len: int) -> map:
        """"""
        return map(lambda term: (term[1], term[0], term[0] + ngrams_len), enumerate(ngrams_iter))
9985
def get_texts(self):
        
        with self.getstream() as text_stream:
            for i, line in enumerate(text_stream):
                line = SMSCorpus.case_normalizer(line)
                if self.mask is not None and not self.mask[i]:
                    continue
                ngrams = []
                for ng in tokens2ngrams(self.tokenizer(line)):
                    if SMSCorpus.ignore_matcher(ng):
                        continue
                    ngrams += [ng]
                yield ngrams
1189612
def ngram_diff(args, parser):
    """"""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    catalogue = utils.get_catalogue(args)
    tokenizer = utils.get_tokenizer(args)
    store.validate(corpus, catalogue)
    if args.asymmetric:
        store.diff_asymmetric(catalogue, args.asymmetric, tokenizer,
                              sys.stdout)
    else:
        store.diff(catalogue, tokenizer, sys.stdout)
836468
def load_corpus(path):
    

    # Check if the data exists, otherwise download or raise
    if not os.path.exists(path):
        raise ValueError((
            "'{}' dataset has not been downloaded, "
            "use the yellowbrick.download module to fetch datasets"
        ).format(path))

    # Read the directories in the directory as the categories.
    categories = [
        cat for cat in os.listdir(path)
        if os.path.isdir(os.path.join(path, cat))
    ]

    files  = [] # holds the file names relative to the root
    data   = [] # holds the text read from the file
    target = [] # holds the string of the category

    # Load the data from the files in the corpus
    for cat in categories:
        for name in os.listdir(os.path.join(path, cat)):
            files.append(os.path.join(path, cat, name))
            target.append(cat)

            with open(os.path.join(path, cat, name), 'r') as f:
                data.append(f.read())


    # Return the data bunch for use similar to the newsgroups example
    return Bunch(
        categories=categories,
        files=files,
        data=data,
        target=target,
    )
1033960
def compute_ngrams(word_list, S=3, T=3):
    
    _ngrams = []
    if isinstance(word_list, str):
        word_list = [word_list]
    for word in word_list:
        for n in range(S, T+1):
            _ngrams += zip(*(word[i:] for i in range(n)))
    return [''.join(_ngram) for _ngram in _ngrams]
553305
def get_category(self, column):
        
        result = pd.Series(index=column.index)

        for category, stats in self.probability_map.items():
            start, end = stats[0]
            result[(start < column) & (column < end)] = category

        return result
489357
def gfm(text):
    """"""
    extractions = {}

    def extract_pre_block(matchobj):
        match = matchobj.group(0)
        hashed_match = hashlib.md5(match.encode('utf-8')).hexdigest()
        extractions[hashed_match] = match
        result = "{gfm-extraction-%s}" % hashed_match
        return result

    def escape_underscore(matchobj):
        match = matchobj.group(0)

        if match.count('_') > 1:
            return re.sub('_', '\_', match)
        else:
            return match

    def newlines_to_brs(matchobj):
        match = matchobj.group(0)
        if re.search("\n{2}", match):
            return match
        else:
            match = match.strip()
            return match + "  \n"

    def insert_pre_block(matchobj):
        string = "\n\n" + extractions[matchobj.group(1)]
        return string

    text = re.sub("(?s)<pre>.*?<\/pre>", extract_pre_block, text)
    text = re.sub("(^(?! {4}|\t)\w+_\w+_\w[\w_]*)", escape_underscore, text)
    text = re.sub("(?m)^[\w\<][^\n]*\n+", newlines_to_brs, text)
    text = re.sub("\{gfm-extraction-([0-9a-f]{32})\}", insert_pre_block, text)

    return text
299129
def word_matches(s1, s2, n=3):
    
    return __matches(s1, s2, word_ngrams, n=n)
1105861
def _str_extract_frame(arr, pat, flags=0):
    
    from pandas import DataFrame

    regex = re.compile(pat, flags=flags)
    groups_or_na = _groups_or_na_fun(regex)
    names = dict(zip(regex.groupindex.values(), regex.groupindex.keys()))
    columns = [names.get(1 + i, i) for i in range(regex.groups)]

    if len(arr) == 0:
        return DataFrame(columns=columns, dtype=object)
    try:
        result_index = arr.index
    except AttributeError:
        result_index = None
    return DataFrame(
        [groups_or_na(val) for val in arr],
        columns=columns,
        index=result_index,
        dtype=object)
35508
public ArrayList<NGram> extract(String text, int maxNumKeywords) {
        ArrayList<String[]> sentences = new ArrayList<>();
        
        SimpleTokenizer tokenizer = new SimpleTokenizer();
        PorterStemmer stemmer = new PorterStemmer();
        
        // Split text into sentences. Stem words by Porter algorithm.
        int ntotal = 0;
        for (String paragraph : SimpleParagraphSplitter.getInstance().split(text)) {
            for (String s : SimpleSentenceSplitter.getInstance().split(paragraph)) {
                String[] sentence = tokenizer.split(s);
                for (int i = 0; i < sentence.length; i++) {
                    sentence[i] = stemmer.stripPluralParticiple(sentence[i]).toLowerCase();
                }
                sentences.add(sentence);
                ntotal += sentence.length;
            }
        }

        //  Extract phrases by Apriori-like algorithm.
        int maxNGramSize = 4;
        ArrayList<NGram> terms = new ArrayList<>();
        AprioriPhraseExtractor phraseExtractor = new AprioriPhraseExtractor();
        for (ArrayList<NGram> ngrams : phraseExtractor.extract(sentences, maxNGramSize, 4)) {
            for (NGram ngram : ngrams) {
                terms.add(ngram);
            }
        }
        Collections.sort(terms);
        
        // Select upto 30% most frequent terms.
        int n = 3 * terms.size() / 10;
        NGram[] freqTerms = new NGram[n];
        for (int i = 0, start = terms.size() - n; i < n; i++) {
            freqTerms[i] = terms.get(start + i);
        }
        
        // Trie for phrase matching.
        Trie<String, Integer> trie = new Trie<>();
        for (int i = 0; i < n; i++) {
            trie.put(freqTerms[i].words, i);
        }
        
        // Build co-occurrence table
        int[] nw = new int[n];
        int[][] table = new int[n][n];
        for (String[] sentence : sentences) {
            Set<Integer> phrases = new HashSet<>();
            for (int j = 1; j <= maxNGramSize; j++) {
                for (int i = 0; i <= sentence.length - j; i++) {
                    String[] phrase  = Arrays.copyOfRange(sentence, i, i+j);
                    Integer index = trie.get(phrase);
                    if (index != null) {
                        phrases.add(index);
                    }
                }
            }
            
            for (int i : phrases) {
                nw[i] += phrases.size();
                for (int j : phrases) {
                    if (i != j) {
                        table[i][j]++;
                    }
                }
            }
        }
        
        // Clustering frequent terms.
        int[] cluster = new int[n];
        for (int i = 0; i < cluster.length; i++) {
            cluster[i] = i;
        }

        //double log2 = Math.log(2.0);
        for (int i = 0; i < n; i++) {
            for (int j = i + 1; j < n; j++) {
                // Mutual information
                if (table[i][j] > 0) {
                    // This doesn't work as ntotal is usually large and thus the mutual information
                    // is way larger than the threshold log2 given in the paper.
                    //double mutual = Math.log((double) ntotal * table[i][j] / (freqTerms[i].freq * freqTerms[j].freq));
                    // Here we just use the (squared) geometric average of co-occurrence probability
                    // It works well to clustering things like "digital computer" and "computer" in practice.
                    double mutual = (double) table[i][j] * table[i][j] / (freqTerms[i].freq * freqTerms[j].freq);
                    if (mutual >= 0.25) {
                        cluster[j] = cluster[i];
                    } /*else {
                        double js = 0.0; // Jsensen-Shannon divergence
                        for (int k = 0; k < n; k++) {
                            double p1 = (double) table[i][k] / freqTerms[i].freq;
                            double p2 = (double) table[j][k] / freqTerms[j].freq;

                            // The formula in the paper is not correct as p is not real probablity.
                            if (p1 > 0 && p2 > 0) {
                                js += -(p1+p2) * Math.log((p1+p2)/2.0) + p1 * Math.log(p1) + p2 * Math.log(p2);
                            }
                        }
                    
                        js /= 2.0;
                        if (js > log2) {
                            cluster[j] = cluster[i];
                        }
                    }*/
                }
            }
        }
        
        // Calculate expected probability
        double[] pc = new double[n];
        for (int i = 0; i < n; i++) {
            for (int j = 0; j < n; j++) {
                pc[cluster[j]] += table[i][j];
            }
        }
        for (int i = 0; i < n; i++) {
            pc[i] /= ntotal;
        }

        
        // Calculate chi-square scores.
        double[] score = new double[n];
        for (int i = 0; i < n; i++) {
            double max = Double.NEGATIVE_INFINITY;
            for (int j = 0; j < n; j++) {
                if (cluster[j] != j) {
                    continue;
                }
                    
                double fwc = 0.0;
                for (int k = 0; k < n; k++) {
                    if (cluster[k] == j)
                    fwc += table[i][k];
                }
                    
                double expected = nw[i] * pc[j];
                double d = (fwc - expected);
                double chisq = d * d / expected;
                score[i] += chisq;
                if (chisq > max) max = chisq;
            }
            //score[i] -= max;
        }
        
        int[] index = QuickSort.sort(score);
        ArrayList<NGram> keywords = new ArrayList<>();
        for (int i = n; i-- > 0; ) {
            boolean add = true;
            // filter out components of phrases, e.g. "digital" in "digital computer".
            for (int j = i+1; j < n; j++) {
                if (cluster[index[j]] == cluster[index[i]]) {
                    if (freqTerms[index[j]].words.length >= freqTerms[index[i]].words.length) {
                        add = false;
                        break;
                    } else {
                        keywords.remove(freqTerms[index[j]]);
                        add = true;
                    }
                }
            }
            
            if (add) {
                keywords.add(freqTerms[index[i]]);
                if (keywords.size() >= maxNumKeywords) break;
            }
        }

        return keywords;
    }