Diagnostic function
    :param string_matrix:
    :return:
    >>> dl = distinct_words([['the', 'quick', 'brown'], ['here', 'lies', 'the', 'fox']])
    >>> sorted(dl)
    ['brown', 'fox', 'here', 'lies', 'quick', 'the']
426104
def distinct_words(string_matrix: List[List[str]]) -> Set[str]:
    
    return set([word
                for sentence in string_matrix
                for word in sentence])
1241304
public Matrix createSparseMatrix() {

        Matrix m = Matrices.create(matrix_row_map.size(), matrix_column_map.size(), false);
        for (int row_num = 0; row_num < matrix_row_map.size(); row_num++) { // for each pattern
            String p = matrix_row_map.get(new Integer(row_num));
            String[] p_sp = p.split(":");
            String a = p_sp[0];
            String b = p_sp[1];
            for (int col_num = 0; col_num < matrix_column_map.size(); col_num++) { // for each phrase
                InterveningWordsPattern col_pattern = matrix_column_map.get(new Integer(col_num));
                String pattern = col_pattern.getPattern();
                String comb_patterns;
                if (col_pattern.getReverse()) { //if the column is a reverse pattern...word2 P word1
                    comb_patterns = ".*\\s" + b + pattern + a + "\\s.*";
                } else {
                    comb_patterns = ".*\\s" + a + pattern + b + "\\s.*";
                }
                try {
                    m.set(row_num, col_num, (double)countWildcardPhraseFrequencies(new File(DATA_DIR), comb_patterns));
                } catch (Exception e) {
                    System.err.println("could not perform wildcard search");
                }
            }
        }
        System.err.println("\nCompleted matrix generation.");
        //System.err.println("Number of rows: " + m.rows());
        //System.err.println("Number of cols: " + m.columns());
        return m;
    }
833370
function(sentences, numberOfTopics, numberOfTermsPerTopic, languages, alphaValue, betaValue, randomSeed) {
    // The result will consist of topics and their included terms [[{"term":"word1", "probability":0.065}, {"term":"word2", "probability":0.047}, ... ], [{"term":"word1", "probability":0.085}, {"term":"word2", "probability":0.024}, ... ]].
    var result = [];
    // Index-encoded array of sentences, with each row containing the indices of the words in the vocabulary.
    var documents = new Array();
    // Hash of vocabulary words and the count of how many times each word has been seen.
    var f = {};
    // Vocabulary of unique words (porter stemmed).
    var vocab=new Array();
    // Vocabulary of unique words in their original form.
    var vocabOrig = {};
    // Array of stop words
    languages = languages || Array('en');

    if (sentences && sentences.length > 0) {
      var stopwords = new Array();

      languages.forEach(function(value) {
          var stopwordsLang = require('./stopwords_' + value + ".js");
          stopwords = stopwords.concat(stopwordsLang.stop_words);
      });

      for(var i=0;i<sentences.length;i++) {
          if (sentences[i]=="") continue;
          documents[i] = new Array();

          var words = sentences[i] ? sentences[i].split(/[\s,\"]+/) : null;

          if(!words) continue;
          for(var wc=0;wc<words.length;wc++) {
              var w=words[wc].toLowerCase().replace(/[^a-z\'A-Z0-9\u00C0-\u00ff ]+/g, '');
              var wStemmed = stem(w);
              if (w=="" || !wStemmed || w.length==1 || stopwords.indexOf(w.replace("'", "")) > -1 || stopwords.indexOf(wStemmed) > -1 || w.indexOf("http")==0) continue;
              if (f[wStemmed]) { 
                  f[wStemmed]=f[wStemmed]+1;
              } 
              else if(wStemmed) { 
                  f[wStemmed]=1; 
                  vocab.push(wStemmed);
                  vocabOrig[wStemmed] = w;
              };
              
              documents[i].push(vocab.indexOf(wStemmed));
          }
      }

      var V = vocab.length;
      var M = documents.length;
      var K = parseInt(numberOfTopics);
      var alpha = alphaValue || 0.1;  // per-document distributions over topics
      var beta = betaValue || .01;  // per-topic distributions over words
      documents = documents.filter((doc) => { return doc.length }); // filter empty documents

      lda.configure(documents,V,10000, 2000, 100, 10, randomSeed);
      lda.gibbs(K, alpha, beta);

      var theta = lda.getTheta();
      var phi = lda.getPhi();

      var text = '';

      //topics
      var topTerms=numberOfTermsPerTopic;
      for (var k = 0; k < phi.length; k++) {
          var things = new Array();
          for (var w = 0; w < phi[k].length; w++) {
               things.push(""+phi[k][w].toPrecision(2)+"_"+vocab[w] + "_" + vocabOrig[vocab[w]]);
          }
          things.sort().reverse();
          //console.log(things);
          if(topTerms>vocab.length) topTerms=vocab.length;

          //console.log('Topic ' + (k + 1));
          var row = [];
          
          for (var t = 0; t < topTerms; t++) {
              var topicTerm=things[t].split("_")[2];
              var prob=parseInt(things[t].split("_")[0]*100);
              if (prob<2) continue;
              
              //console.log('Top Term: ' + topicTerm + ' (' + prob + '%)');
              
              var term = {};
              term.term = topicTerm;
              term.probability = parseFloat(things[t].split("_")[0]);
              row.push(term);
          }

          result.push(row);
      }
    }
    
    return result;
}
792267
def surround_previous_word(input_str):
    '''
    
    '''
    start = None
    end = None
    for i, char in enumerate(reversed(input_str)):
        if start is None:
            if char in '{}()[]<>?|':
                return input_str
            elif char != ' ':
                start = i
        else:
            if char in '{}()[]<>?| ':
                end = i
                break
    if start is None:
        return input_str
    if end is None:
        end = len(input_str)
    new_str = ''
    for i, char in enumerate(reversed(input_str)):
        if char == ' ' and i + 1 == start:
            continue
        if i == start:
            new_str += ') '
        elif i == end:
            new_str += '('
        new_str += char
    if end == len(input_str):
        new_str += '('
    return new_str[::-1]
1197455
def _compute_term_frequency(self, matrix, smooth=0.4):
        
        assert 0.0 <= smooth < 1.0

        max_word_frequencies = numpy.max(matrix, axis=0)
        rows, cols = matrix.shape
        for row in range(rows):
            for col in range(cols):
                max_word_frequency = max_word_frequencies[col]
                if max_word_frequency != 0:
                    frequency = matrix[row, col]/max_word_frequency
                    matrix[row, col] = smooth + (1.0 - smooth)*frequency

        return matrix
944456
def short_text__str(self, column_name, output_column_prefix):
        

        from ._ngram_counter import NGramCounter
        from ._tfidf import TFIDF

        return [NGramCounter(features=[column_name],
                             n = 3,
                             method = "character",
                             output_column_prefix = output_column_prefix),

                 TFIDF(features=[column_name],
                       min_document_frequency=0.01,
                       max_document_frequency=0.5,
                       output_column_prefix = output_column_prefix)]
339302
def matrix_str(p, decimal_places=2, print_zero=True, label_columns=False):
    ''''''
    return '[{0}]'.format("\n  ".join([(str(i) if label_columns else '') + vector_str(a, decimal_places, print_zero) for i, a in enumerate(p)]))
733186
def dedent_block_string_value(raw_string: str) -> str:
    
    lines = raw_string.splitlines()

    common_indent = None
    for line in lines[1:]:
        indent = leading_whitespace(line)
        if indent < len(line) and (common_indent is None or indent < common_indent):
            common_indent = indent
        if common_indent == 0:
            break

    if common_indent:
        lines[1:] = [line[common_indent:] for line in lines[1:]]

    while lines and not lines[0].strip():
        lines = lines[1:]

    while lines and not lines[-1].strip():
        lines = lines[:-1]

    return "\n".join(lines)
197088
def canonical_chrom_sorted(in_chroms):
    
    if len(in_chroms) == 0:
        return []
    chr_prefix = False
    mt = False
    if in_chroms[0].startswith('chr'):
        in_chroms = [x.lstrip('chr') for x in in_chroms]
        chr_prefix = True
    if 'MT' in in_chroms:
        in_chroms[in_chroms.index('MT')] = 'M'
        mt = True
    in_chroms = sorted(in_chroms, key=lambda c: int(c) if c not in ('X', 'Y', 'M') else c)
    try:
        m_index = in_chroms.index('M')
    except ValueError:
        pass
    else:
        in_chroms.pop(m_index)
        in_chroms.append('M')
    # At this point it should be nicely sorted
    if mt:
        in_chroms[in_chroms.index('M')] = 'MT'
    if chr_prefix:
        in_chroms = [''.join(['chr', x]) for x in in_chroms]
    return in_chroms
219348
def string_to_list(string, sep=",", filter_empty=False):
    """"""
    return [value.strip() for value in string.split(sep)
            if (not filter_empty or value)]
687181
def get_multi_word_keywords(features):
        
        keys = {
            'is not': Token(TokenTypes.NOT_EQUAL, 'is not'),
        }
        return OrderedDict(sorted(list(keys.items()), key=lambda t: len(t[0]), reverse=True))
1326606
def unique_list(input_list):
    
    """
    output_list = []
    if len(input_list) > 0:
        dim = _sp.shape(input_list)[1]
        for i in input_list:
            match = False
            for j in output_list:
                if dim == 3:
                    if i[0] == j[0] and i[1] == j[1] and i[2] == j[2]:
                        match = True
                elif dim == 2:
                    if i[0] == j[0] and i[1] == j[1]:
                        match = True
                elif dim == 1:
                    if i[0] == j[0]:
                        match = True
            if match is False:
                output_list.append(i)
    return output_list
85752
def term_rank(self, items, print_output=True):

        

        if items == 'stems':
            v = self.stems
        elif items == 'tokens':
            v = self.tokens
        elif items == 'bigrams':
            v = self.bigrams
        else:
            raise ValueError("Items must be either \'tokens\' , \'bigrams\' or \'stems\'.")

        agg = itertools.chain(*v)
        counts = collections.Counter(agg)

        v_unique = list(map(lambda x: set(x), v))
        agg_d = itertools.chain(*v_unique)
        counts_d = collections.Counter(agg_d)

        unique_tokens = set(itertools.chain(*v))

        def tf_idf_compute(t):
            return (1 + np.log(counts[t]))*np.log(self.N/counts_d[t])

        unsorted_df = [counts[t] for t in unique_tokens]
        unsorted_tf_idf = [tf_idf_compute(t) for t in unique_tokens]

        self.df_ranking = sorted(zip(unique_tokens, unsorted_df),
                                 key=lambda x: x[1], reverse=True)
        self.tfidf_ranking = sorted(zip(unique_tokens, unsorted_tf_idf),
                                    key=lambda x: x[1], reverse=True)

        if print_output:
            with codecs.open('df_ranking.csv', 'w', 'utf-8') as f:
                    for p in self.df_ranking:
                        f.write("%s,%d\n" % (p[0], p[1]))
            with codecs.open('tfidf_ranking.csv', 'w', 'utf-8') as f:
                    for p in self.tfidf_ranking:
                        f.write("%s,%f\n" % (p[0], p[1]))
4237
def str2dict_values(str_in):
    '''
    
    '''
    tmp_dict = str2dict(str_in)
    if tmp_dict is None:
        return None
    return [tmp_dict[key] for key in sorted(k for k in tmp_dict)]
1046419
def match_list(query_list, string):
    
    # Get rid of 'the' word to ease string matching
    match = False
    index = 0
    string = ' '.join(filter_stopwords(string))

    if not isinstance(query_list, list):
        query_list = [query_list]

    while index < len(query_list):
        query = query_list[index]
        words_query = filter_stopwords(query)
        match = all(word in string for word in words_query)
        if match:
            break

        index += 1

    return match
1065803
def select_string(self, rows: List[Row], column: StringColumn) -> List[str]:
        
        return [str(row.values[column.name]) for row in rows if row.values[column.name] is not None]
857124
def get_top_n_meanings(strings, n):
    
    scored_strings = [(s, score_meaning(s)) for s in strings]
    scored_strings.sort(key=lambda tup: -tup[1])
    return scored_strings[:n]