TODO: better algorithm to assign furigana to parts of the word
544287
public List<SubtitleItem.Inner> createElementsList(String text,
			String furigana, SubtitleItem item) {

		ArrayList<SubtitleItem.Inner> list = new ArrayList<SubtitleItem.Inner>();
		String kanjiChars = null;
		String chars = null;
		boolean hasKanji = false;
		for (int i = 0; i < text.length(); i++) {
			char c = text.charAt(i);
			boolean isKanji = isKanjiChar(c);
			// the second check is useful to process words with kanji and
			// katakana
			if (isKanji || furigana.indexOf(c) < 0) {
				if (isKanji) {
					// use furigana only if there is at least one kanji in the
					// word
					hasKanji = true;
				}
				if (chars != null) {
					addText(list, chars);
					chars = null;
				}
				if (kanjiChars == null) {
					kanjiChars = Character.toString(c);
				} else {
					kanjiChars += c;
				}
			} else {
				if (kanjiChars != null) {
					addKanji(list, kanjiChars);
					kanjiChars = null;
				}
				if (chars == null) {
					chars = Character.toString(c);
				} else {
					chars += c;
				}
			}
		}
		if (kanjiChars != null) {
			addKanji(list, kanjiChars);
		}
		if (hasKanji) {
			if (chars != null) {
				addText(list, chars);
			}
			furiganaToElements(new StringBuilder(furigana), list);
			return list;
		}
		return null;
	}
1358946
def tokenize_words(self, text):
        """"""
        return [
            self.strip_punctuation(word) for word in text.split(' ')
            if self.strip_punctuation(word)
        ]
105852
function getAllEmojiForWord(originalWord) {
  let word = originalWord.trim().toLowerCase();

  if (!word || word === '' || word === 'a' || word === 'it' || word === 'is')
    return '';

  // Maybe this is a plural word but the word is the singular?
  // Don't do it for two letter words since "as" would become "a" etc.
  let maybeSingular = '';
  if (word.length > 2 && word[word.length - 1] == 's') {
    maybeSingular = word.slice(0, word.length - 1);
  }

  // Maybe this is a singular word but the word is the plural?
  // Don't do this for single letter since that will pluralize crazy things.
  let maybePlural = (word.length == 1) ? '' : word + 's';

  let maybeVerbedSimple = '';
  let maybeVerbedVowel = '';
  let maybeVerbedDoubled  = '';

  if (word.indexOf('ing') !== -1) {
    let verb = word.substr(0, word.length - 3);
    // eating -> eat
    maybeVerbedSimple = verb;
    // dancing -> dance
    maybeVerbedVowel = verb + 'e';
    // running -> run
    maybeVerbedDoubled = verb.substr(0, verb.length - 1);
  }

  // Go through all the things and find the first one that matches.
  let useful = [];

  // If this is already an emoji, don't try to translate it.
  if (isMaybeAlreadyAnEmoji(word)) {
    useful.push(word);
    return useful;
  }

  // If it's "i" or "i", add some faces to it.
  if (word === 'i' || word === 'you') {
    useful.push('😀');
    useful.push('😊');
  } else if (word === 'she'){
    useful.push('💁');
  } else if (word === 'he'){
    useful.push('💁‍♂️');
  } else if (word === 'we' || word === 'they') {
    useful.push('👩‍👩‍👦‍👦');
  } else if (word === 'am' || word === 'is' || word === 'are') {
    useful.push('👉');
  } else if (word === 'thanks') {
    useful.push('🙌');
  }

  for (let emoji in allEmoji) {
    let words = allEmoji[emoji].keywords;
    // TODO: omg refactor this one day, please. Why is this even. Why.
    if (word == allEmoji[emoji].char ||
        emoji == word || (emoji == word + '_face') ||
        emoji == maybeSingular || emoji == maybePlural ||
        emoji == maybeVerbedSimple || emoji == maybeVerbedVowel || emoji == maybeVerbedDoubled ||
        (words && words.indexOf(word) >= 0) ||
        (words && words.indexOf(maybeSingular) >= 0) ||
        (words && words.indexOf(maybePlural) >= 0) ||
        (words && words.indexOf(maybeVerbedSimple) >= 0) ||
        (words && words.indexOf(maybeVerbedVowel) >= 0) ||
        (words && words.indexOf(maybeVerbedDoubled) >= 0)) {
      // If it's a two letter word that got translated to a flag, it's 99% of the
      // time incorrect, so stop doing that.
      if (!(word.length <= 3 && allEmoji[emoji].category == 'flags')) {
        useful.push(allEmoji[emoji].char);
      }
    }
  }
  return (useful.length === 0) ? '' : useful;
}
108772
function getEmojiForWord(word) {
  let translations = getAllEmojiForWord(word);
  return translations[Math.floor(Math.random() * translations.length)];
}
340836
private void trimZeros()
  {
    // loop over ALL_ZEROS_LITERAL words
    int w;
    do {
      w = words[lastWordIndex];
      if (w == ConciseSetUtils.ALL_ZEROS_LITERAL) {
        lastWordIndex--;
      } else if (isZeroSequence(w)) {
        if (simulateWAH || isSequenceWithNoBits(w)) {
          lastWordIndex--;
        } else {
          // convert the sequence in a 1-bit literal word
          words[lastWordIndex] = getLiteral(w);
          return;
        }
      } else {
        // one sequence or literal
        return;
      }
      if (lastWordIndex < 0) {
        reset();
        return;
      }
    } while (true);
  }
119177
private ArrayList<String> ngram(List tokens, int[] gramSizes2) {
        ArrayList<String> list = new ArrayList<String>();
        StringBuffer buf = new StringBuffer();
        for (int j = 0; j < gramSizes.length; j++) {
            int len = gramSizes[j];
            if (len <= 0 || len > tokens.size())
                continue;
            for (int i = 0; i < tokens.size() - len+1; i++) {
                buf.delete(0, buf.length());
                int k = 0;
                for(; k < len-1; ++k)   {
                    buf.append(tokens.get(i+k));
                    buf.append(' ');
                }
                buf.append(tokens.get(i+k));
                list.add(buf.toString().intern());
            }
        }
        return list;
    }
780809
def mixedcase(path):
    """"""
    words = path.split('_')
    return words[0] + ''.join(word.title() for word in words[1:])
975327
function trim(str) {
  var input = str.replace(/\s\s+/g, ' ').trim();

  var words = input.split(' ');
  if (_.contains(LANG_WORDS, _.first(words))) {
    return words.slice(1).join(' ');
  }
  return input;
}
459682
def _normalize_tags(chunk):
    
    ret = []
    for word, tag in chunk:
        if tag == 'NP-TL' or tag == 'NP':
            ret.append((word, 'NNP'))
            continue
        if tag.endswith('-TL'):
            ret.append((word, tag[:-3]))
            continue
        if tag.endswith('S'):
            ret.append((word, tag[:-1]))
            continue
        ret.append((word, tag))
    return ret
1268302
private void splitToken(final PropertyPath tokenStore, final String path) {
        
        // 解析時の途中の文字を一時的に補完しておく。
        final LinkedList<String> stack = new LinkedList<String>();
        
        final int length = path.length();
        for(int i=0; i < length; i++) {
            final char c = path.charAt(i);
            
            if(StackUtils.equalsTopElement(stack, STR_ESCAPE_CHAR)) {
                // スタックの一番上がエスケープの文字の場合、通常の文字として扱う。
                stack.push(String.valueOf(c));
                
            } else if (c == '.') {
                
                if(StackUtils.equalsBottomElement(stack, "[")) {
                    // キーの囲み文字の途中の場合、文字列としてスタックに積む。
                    stack.push(String.valueOf(c));
                    
                } else if(!stack.isEmpty()) {
                    // 既に文字が入っている場合は、既存のものを取り出し分割する。
                    tokenStore.add(new Token.Name(StackUtils.popupAndConcat(stack).trim()));
                    tokenStore.add(new Token.Separator());
                    
                } else {
                    tokenStore.add(new Token.Separator());
                }
                
            } else if(c == '[') {
                
                if(!stack.isEmpty()) {
                    // 既に文字が入っている場合は、既存のものを取り出し分割する。
                    tokenStore.add(new Token.Name(StackUtils.popupAndConcat(stack).trim()));
                }
                
                stack.push(String.valueOf(c));
                
            } else if(c == ']') {
                if(StackUtils.equalsBottomElement(stack, "[")) {
                    // 条件の終わりの場合
                    tokenStore.add(new Token.Key(StackUtils.popupAndConcat(stack) + c));
                    
                } else {
                    stack.push(String.valueOf(c));
                }
                
            } else if(c == ' ') {
                // キーの囲み文字の中であれば、空白は無視しない。
                if(StackUtils.equalsBottomElement(stack, "[")) {
                    stack.push(String.valueOf(c));
                }
                
            } else {
                stack.push(String.valueOf(c));
            }
            
        }
        
        if(!stack.isEmpty()) {
            tokenStore.add(new Token.Name(StackUtils.popupAndConcat(stack).trim()));
        }
        
    }
816085
function( term ) {
        var split_term = term.split(' ');
        var matchers = [];
        for (var i=0; i < split_term.length; i++) {
          if ( split_term[i].length > 0 ) {
            var matcher = {};
            matcher['partial'] = new RegExp( $.ui.autocomplete.escapeRegex( split_term[i] ), "i" );
            if ( context.settings['relevancy-sorting'] ) {
              matcher['strict'] = new RegExp( "^" + $.ui.autocomplete.escapeRegex( split_term[i] ), "i" );
            }
            matchers.push( matcher );
          }
        };

        return $.grep( context.options, function( option ) {
          var partial_matches = 0;
          if ( context.settings['relevancy-sorting'] ) {
            var strict_match = false;
            var split_option_matches = option.matches.split(' ');
          }
          for ( var i=0; i < matchers.length; i++ ) {
            if ( matchers[i]['partial'].test( option.matches ) ) {
              partial_matches++;
            }
            if ( context.settings['relevancy-sorting'] ) {
              for (var q=0; q < split_option_matches.length; q++) {
                if ( matchers[i]['strict'].test( split_option_matches[q] ) ) {
                  strict_match = true;
                  break;
                }
              };
            }
          };
          if ( context.settings['relevancy-sorting'] ) {
            var option_score = 0;
            option_score += partial_matches * context.settings['relevancy-sorting-partial-match-value'];
            if ( strict_match ) {
              option_score += context.settings['relevancy-sorting-strict-match-value'];
            }
            option_score = option_score * option['relevancy-score-booster'];
            option['relevancy-score'] = option_score;
          }
          return (!term || matchers.length === partial_matches );
        });
      }
332987
function step1b(token) {
    var r1 = getR1(token);

    if (!r1) {
        return token;
    }

    if (token.match(/(b|c|d|f|g|h|j|l|m|n|o|p|r|t|v|y|z)s$/)) {
        return token.slice(0, -1);
    }

    if (token.match(/([^aeiouyæåø]k)s$/)) {
        return token.slice(0, -1);
    }

    return token;
}
26607
def slugify(text, length_limit=0, delimiter=u'-'):
    """"""
    result = []
    for word in _punctuation_regex.split(text.lower()):
        word = _available_unicode_handlers[0](word)
        if word:
            result.append(word)
    slug = delimiter.join(result)
    if length_limit > 0:
        return slug[0:length_limit]
    return slug
395010
function tokenize(content, rules = parseOrder) {
    let data = []
    if(content) {
        let lines = content.split(/(\r\n|\n)/)
        for (let idx = 0; idx < lines.length; idx++) {
            for (let i = 0; i < lines[idx].length;) {
                let chunk = lines[idx].slice(i)
                let match = rules.some(item => {
                    if (item.pattern.test(chunk)) {
                        let matchObject = chunk.match(item.pattern)
                        i += matchObject[0].length
                        data.push([ item.label, matchObject[0] ])
                        return true
                    }
                })
                if (!match) {
                    data.push([ UNKNOWN, chunk[i] ])
                    i++
                }
            }
        }
    }
    return data
}
1080553
private Map<String, Double> distance1Generation(String word) {
        if (word == null || word.length() < 1) throw new RuntimeException("Input words Error: " + word);

        Map<String, Double> result = new HashMap<String, Double>();

        String prev;
        String last;

        for (int i = 0; i < word.length(); i++) {
            // Deletion
            prev = word.substring(0, i);
            last = word.substring(i + 1, word.length());
            result.put(prev + last, 1.0);

            // transposition
            if ((i + 1) < word.length()) {
                prev = word.substring(0, i);
                last = word.substring(i + 2, word.length());
                String trans = prev + word.charAt(i + 1) + word.charAt(i) + last;
                result.put(trans, 1.0);
            }

            // alter
            prev = word.substring(0, i);
            last = word.substring(i + 1, word.length());
            for (int j = 0; j < 26; j++) {
                result.put(prev + (char) (j + 97) + last, 1.0);
            }

            // insertion
            prev = word.substring(0, i);
            last = word.substring(i + 1, word.length());
            for (int j = 0; j < 26; j++) {
                result.put(prev + (char) (j + 97) + word.charAt(i) + last, 1.0);
                result.put(prev + word.charAt(i) + (char) (j + 97) + last, 1.0);
            }

        }

        result.remove(word);
        return result;
    }
1103975
function keywords( words ) {
	var len = words.length,
		n = len - 1,
		out,
		i;

	out = '';
	for ( i = 0; i < len; i++ ) {
		out += '\n\t\t';
		out += '\"' + words[ i ] + '\"';
		if ( i === n ) {
			out += '\n\t';
		} else {
			out += ',';
		}
	}
	return out;
}
558350
def join(self, words):
        """"""
        chars = list(u"".join(words))
        start = True

        for i in xrange(len(chars)):
            char = chars[i]
            if char.isalpha():
                if start:
                    chars[i] = char.upper()
                else:
                    chars[i] = char.lower()

                start = False
            else:
                if i > 2 and chars[i - 1] in ".?!" and char.isspace():
                    start = True

        return u"".join(chars)