Convert katakana to romaji with Lucene converter if Kurikosu throws
errors
436308
private static String convertToRomaji(String katakana)
			throws EasyJaSubException {
		try {
			return Kurikosu.convertKatakanaToRomaji(katakana);
		} catch (EasyJaSubException ex) {
			try {
				return LuceneUtil.katakanaToRomaji(katakana);
			} catch (Throwable luceneEx) {
				throw ex;
			}
		}
	}
1252837
function readFileOrError(uri) {
    var content;
    try {
        content = fs.readFileSync(uri, 'utf8');
    } catch (err) {
        return Result.Err(err);
    }

    return Result.Ok(content);
}
13053
function fontDetect(content, fallback_font_type, options = {}){
  if (!content) {
    if (!globalOptions.isSilentMode()) console.warn('Content must be specified on knayi.fontDetect.');
    return fallback_font_type || 'en';
  }

	if (content === '')
		return content;

	if (!mmCharacterRange.test(content))
		return fallback_font_type || 'en';

	content = content.trim().replace(/\u200B/g, '');
	fallback_font_type = fallback_font_type || 'zawgyi';

  options = globalOptions.detector(options);

  if (options.use_myanmartools) {
    var myanmartools_zg_probability = myanmartoolZawgyiDetector.getZawgyiProbability(content);

    if (myanmartools_zg_probability < options.myanmartools_zg_threshold[0]) {
  		return 'unicode';
  	} else if (myanmartools_zg_probability > options.myanmartools_zg_threshold[1]) {
  		return 'zawgyi';
  	} else {
  		return fallback_font_type;
  	}

  } else {
    var match = {};

    for (var type in library.detect) {
  		match[type] = 0;

  		for (var i = 0; i < library.detect[type].length; i++) {
  			var rule = library.detect[type][i]
  			var m = content.match(rule);
  			match[type] += (m && m.length) || 0;
  		}
  	}

    if (match.unicode > match.zawgyi) {
  		return 'unicode';
  	} else if (match.unicode < match.zawgyi) {
  		return 'zawgyi';
  	} else {
  		return fallback_font_type;
  	}

  }
}
1099222
function replace(value) {

  // 漢数字、漢字、全角文字を半角数字に変換
  value = suji(value);

  // 和暦時西暦に変換
  value = value.replace(REG_ERA, (x, g, n, af) => {
    var era = ERA_MAP[g];
    return '' + (era.y + (n === '元' ?  1 : +n) - 1) + af;
  });

  // 午前午後を24時間表記に変換
  value = value.replace(REG_AMPM, (x, ampm, n) => {
    var pm = ampm === '午後' || ampm.toLowerCase() === 'pm';
    return pm ? +n + 12 : n;
  });

  return value;
}
637130
function getSlug(input, opts) {
        var separator = '-';
        var result = '';
        var diatricString = '';
        var convertSymbols = true;
        var customReplacements = {};
        var maintainCase;
        var titleCase;
        var truncate;
        var uricFlag;
        var uricNoSlashFlag;
        var markFlag;
        var symbol;
        var langChar;
        var lucky;
        var i;
        var ch;
        var l;
        var lastCharWasSymbol;
        var lastCharWasDiatric;
        var allowedChars = '';

        if (typeof input !== 'string') {
            return '';
        }

        if (typeof opts === 'string') {
            separator = opts;
        }

        symbol = symbolMap.en;
        langChar = langCharMap.en;

        if (typeof opts === 'object') {
            maintainCase = opts.maintainCase || false;
            customReplacements = (opts.custom && typeof opts.custom === 'object') ? opts.custom : customReplacements;
            truncate = (+opts.truncate > 1 && opts.truncate) || false;
            uricFlag = opts.uric || false;
            uricNoSlashFlag = opts.uricNoSlash || false;
            markFlag = opts.mark || false;
            convertSymbols = (opts.symbols === false || opts.lang === false) ? false : true;
            separator = opts.separator || separator;

            if (uricFlag) {
                allowedChars += uricChars;
            }

            if (uricNoSlashFlag) {
                allowedChars += uricNoSlashChars;
            }

            if (markFlag) {
                allowedChars += markChars;
            }

            symbol = (opts.lang && symbolMap[opts.lang] && convertSymbols) ?
                symbolMap[opts.lang] : (convertSymbols ? symbolMap.en : {});

            langChar = (opts.lang && langCharMap[opts.lang]) ?
                langCharMap[opts.lang] :
                opts.lang === false || opts.lang === true ? {} : langCharMap.en;

            // if titleCase config is an Array, rewrite to object format
            if (opts.titleCase && typeof opts.titleCase.length === 'number' && Array.prototype.toString.call(opts.titleCase)) {
                opts.titleCase.forEach(function (v) {
                    customReplacements[v + ''] = v + '';
                });

                titleCase = true;
            } else {
                titleCase = !!opts.titleCase;
            }

            // if custom config is an Array, rewrite to object format
            if (opts.custom && typeof opts.custom.length === 'number' && Array.prototype.toString.call(opts.custom)) {
                opts.custom.forEach(function (v) {
                    customReplacements[v + ''] = v + '';
                });
            }

            // custom replacements
            Object.keys(customReplacements).forEach(function (v) {
                var r;

                if (v.length > 1) {
                    r = new RegExp('\\b' + escapeChars(v) + '\\b', 'gi');
                } else {
                    r = new RegExp(escapeChars(v), 'gi');
                }

                input = input.replace(r, customReplacements[v]);
            });

            // add all custom replacement to allowed charlist
            for (ch in customReplacements) {
                allowedChars += ch;
            }
        }

        allowedChars += separator;

        // escape all necessary chars
        allowedChars = escapeChars(allowedChars);

        // trim whitespaces
        input = input.replace(/(^\s+|\s+$)/g, '');

        lastCharWasSymbol = false;
        lastCharWasDiatric = false;

        for (i = 0, l = input.length; i < l; i++) {
            ch = input[i];

            if (isReplacedCustomChar(ch, customReplacements)) {
                // don't convert a already converted char
                lastCharWasSymbol = false;
            } else if (langChar[ch]) {
                // process language specific diactrics chars conversion
                ch = lastCharWasSymbol && langChar[ch].match(/[A-Za-z0-9]/) ? ' ' + langChar[ch] : langChar[ch];

                lastCharWasSymbol = false;
            } else if (ch in charMap) {
                // the transliteration changes entirely when some special characters are added
                if (i + 1 < l && lookAheadCharArray.indexOf(input[i + 1]) >= 0) {
                    diatricString += ch;
                    ch = '';
                } else if (lastCharWasDiatric === true) {
                    ch = diatricMap[diatricString] + charMap[ch];
                    diatricString = '';
                } else {
                    // process diactrics chars
                    ch = lastCharWasSymbol && charMap[ch].match(/[A-Za-z0-9]/) ? ' ' + charMap[ch] : charMap[ch];
                }

                lastCharWasSymbol = false;
                lastCharWasDiatric = false;
            } else if (ch in diatricMap) {
                diatricString += ch;
                ch = '';
                // end of string, put the whole meaningful word
                if (i === l - 1) {
                    ch = diatricMap[diatricString];
                }
                lastCharWasDiatric = true;
            } else if (
                // process symbol chars
                symbol[ch] && !(uricFlag && uricChars
                    .indexOf(ch) !== -1) && !(uricNoSlashFlag && uricNoSlashChars
                    // .indexOf(ch) !== -1) && !(markFlag && markChars
                    .indexOf(ch) !== -1)) {
                ch = lastCharWasSymbol || result.substr(-1).match(/[A-Za-z0-9]/) ? separator + symbol[ch] : symbol[ch];
                ch += input[i + 1] !== void 0 && input[i + 1].match(/[A-Za-z0-9]/) ? separator : '';

                lastCharWasSymbol = true;
            } else {
                if (lastCharWasDiatric === true) {
                    ch = diatricMap[diatricString] + ch;
                    diatricString = '';
                    lastCharWasDiatric = false;
                } else if (lastCharWasSymbol && (/[A-Za-z0-9]/.test(ch) || result.substr(-1).match(/A-Za-z0-9]/))) {
                    // process latin chars
                    ch = ' ' + ch;
                }
                lastCharWasSymbol = false;
            }

            // add allowed chars
            result += ch.replace(new RegExp('[^\\w\\s' + allowedChars + '_-]', 'g'), separator);
        }

        if (titleCase) {
            result = result.replace(/(\w)(\S*)/g, function (_, i, r) {
                var j = i.toUpperCase() + (r !== null ? r : '');
                return (Object.keys(customReplacements).indexOf(j.toLowerCase()) < 0) ? j : j.toLowerCase();
            });
        }

        // eliminate duplicate separators
        // add separator
        // trim separators from start and end
        result = result.replace(/\s+/g, separator)
            .replace(new RegExp('\\' + separator + '+', 'g'), separator)
            .replace(new RegExp('(^\\' + separator + '+|\\' + separator + '+$)', 'g'), '');

        if (truncate && result.length > truncate) {
            lucky = result.charAt(truncate) === separator;
            result = result.slice(0, truncate);

            if (!lucky) {
                result = result.slice(0, result.lastIndexOf(separator));
            }
        }

        if (!maintainCase && !titleCase) {
            result = result.toLowerCase();
        }

        return result;
    }
1331039
private I18nReader getI18nReader(File sourceFile) throws MojoFailureException {
    String extension = FileUtils.getExtension(sourceFile.getName());
    if (StringUtils.equalsIgnoreCase(extension, FILE_EXTENSION_PROPERTIES)) {
      return new PropertiesI18nReader();
    }
    if (StringUtils.equalsIgnoreCase(extension, FILE_EXTENSION_XML)) {
      return new XmlI18nReader();
    }
    if (StringUtils.equalsIgnoreCase(extension, FILE_EXTENSION_JSON)) {
      return new JsonI18nReader();
    }
    throw new MojoFailureException("Unsupported file extension '" + extension + "': " + sourceFile.getAbsolutePath());
  }
863110
def t_ARTICLEHEADER(self, token):
        # \xef\xbc\x9a is the "fullwidth colon" used in Japanese for instance
        '
        number = token.lexer.lexmatch.group("number").decode("utf8")
        newtag = token.lexer.lexmatch.group("newtag").decode("utf8")
        oldtag = token.lexer.lexmatch.group("oldtag").decode("utf8")
        name = token.lexer.lexmatch.group("name").decode("utf8")
        sep = token.lexer.lexmatch.group("sep").decode("utf8")
        title = token.lexer.lexmatch.group("title").decode("utf8")
        token.value = (number, newtag, oldtag, name, title, sep)
        token.lexer.lineno += 1
        return token
1366053
function(callback) {
        dataIndex = {};

        var pos = 0;

        enamdictData.split("\n").forEach(function(line) {
            var parts = line.split("|");
            var cleanName = parts[0];
            var romaji = parts[1];

            if (!(cleanName in dataIndex)) {
                dataIndex[cleanName] = pos;
            }

            if (!(romaji in dataIndex)) {
                dataIndex[romaji] = pos;
            }

            // +1 for the \n
            pos += line.length + 1;
        });

        callback(null, this);
    }
907123
def toJSON(self, use_unicode=True):
        
        rdd = self._jdf.toJSON()
        return RDD(rdd.toJavaRDD(), self._sc, UTF8Deserializer(use_unicode))
1328138
def native_(s, encoding='latin-1', errors='strict'):
    '''
    
    '''
    if PY3:
        out = s if isinstance(s, text_type) else str(s, encoding, errors)
    else:
        out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s)

    return out
1268302
private void splitToken(final PropertyPath tokenStore, final String path) {
        
        // 解析時の途中の文字を一時的に補完しておく。
        final LinkedList<String> stack = new LinkedList<String>();
        
        final int length = path.length();
        for(int i=0; i < length; i++) {
            final char c = path.charAt(i);
            
            if(StackUtils.equalsTopElement(stack, STR_ESCAPE_CHAR)) {
                // スタックの一番上がエスケープの文字の場合、通常の文字として扱う。
                stack.push(String.valueOf(c));
                
            } else if (c == '.') {
                
                if(StackUtils.equalsBottomElement(stack, "[")) {
                    // キーの囲み文字の途中の場合、文字列としてスタックに積む。
                    stack.push(String.valueOf(c));
                    
                } else if(!stack.isEmpty()) {
                    // 既に文字が入っている場合は、既存のものを取り出し分割する。
                    tokenStore.add(new Token.Name(StackUtils.popupAndConcat(stack).trim()));
                    tokenStore.add(new Token.Separator());
                    
                } else {
                    tokenStore.add(new Token.Separator());
                }
                
            } else if(c == '[') {
                
                if(!stack.isEmpty()) {
                    // 既に文字が入っている場合は、既存のものを取り出し分割する。
                    tokenStore.add(new Token.Name(StackUtils.popupAndConcat(stack).trim()));
                }
                
                stack.push(String.valueOf(c));
                
            } else if(c == ']') {
                if(StackUtils.equalsBottomElement(stack, "[")) {
                    // 条件の終わりの場合
                    tokenStore.add(new Token.Key(StackUtils.popupAndConcat(stack) + c));
                    
                } else {
                    stack.push(String.valueOf(c));
                }
                
            } else if(c == ' ') {
                // キーの囲み文字の中であれば、空白は無視しない。
                if(StackUtils.equalsBottomElement(stack, "[")) {
                    stack.push(String.valueOf(c));
                }
                
            } else {
                stack.push(String.valueOf(c));
            }
            
        }
        
        if(!stack.isEmpty()) {
            tokenStore.add(new Token.Name(StackUtils.popupAndConcat(stack).trim()));
        }
        
    }
713111
public synchronized void registerDefaultFormat() {
        
        
        final Locale[] availableLocales = new Locale[]{Locale.JAPANESE};
        
        
        // 組み込み書式の登録
        for(int i=0; i <= 58; i++) {
            
            final CellFormatter formatter = createDefaultFormatter(String.valueOf(i), availableLocales);
            if(formatter != null) {
                registerFormatter((short) i, formatter);
                
            }
            
        }
        
        // 特別な書式
        final String[] names = new String[]{"F800", "F400"};
        for(String name : names) {
            
            final String key = String.format("format.%s", name);
            
            final String defaultFormat = messageResolver.getMessage(key);
            if(defaultFormat == null) {
                continue;
            }
            
            final CellFormatter formatter = createDefaultFormatter(name, availableLocales);
            if(formatter != null) {
                registerFormatter(defaultFormat, formatter);
            }
        }
        
    }
245897
function getType(input) {
  switch (true) {
    case (isCharJapanesePunctuation(input)): return 'japanesePunctuation';
    case (isCharKanji(input)): return 'kanji';
    case (isCharHiragana(input)): return 'hiragana';
    case (isCharKatakana(input)): return 'katakana';
    default: return 'romaji';
  }
}
1116276
public static void katakanaToRomaji(Appendable builder, CharSequence s)
			throws IOException {
		ToStringUtil.getRomanization(builder, s);
	}
612970
private List<Token> convertFactor(final Token.Factor factor) {
        
        final String item = factor.getValue();
        final int itemLength = item.length();
        
        final List<Token> list = new ArrayList<>();
        
        int idx = 0;
        StringBuilder noTermChar = new StringBuilder(); // フォーマット以外の文字列を積む。
        while(idx < itemLength) {
            
            String matchChars = null;
            for(String chars : SORTED_DATE_CHARS) {
                if(Utils.startsWithIgnoreCase(item, chars, idx)) {
                    matchChars = item.substring(idx, idx + chars.length());
                    break;
                }
            }
            
            if(matchChars == null) {
                // フォーマットでない場合は、文字列としてバッファに追加する。
                noTermChar.append(item.charAt(idx));
                idx++;
            } else {
                if(noTermChar.length() > 0) {
                    // 今まで積んだバッファを、文字列として分割する。
                    list.add(Token.factor(noTermChar.toString()));
                    noTermChar = new StringBuilder();
                }
                
                list.add(Token.formatter(matchChars));
                idx += matchChars.length();
                
            }
        }
        
        if(noTermChar.length() > 0) {
            list.add(Token.factor(noTermChar.toString()));
        }
        
        return list;
    }
789785
public T read() throws IOException {
        
        if(readRow()) {
            
            final T bean = instantiateBean(beanMappingCache.getOriginal().getType());
            final CsvBindingErrors bindingErrors = new CsvBindingErrors(beanMappingCache.getOriginal().getType());
            
            final CsvContext context = new CsvContext(getLineNumber(), getRowNumber(), 1);
            context.setRowSource(new ArrayList<Object>(processedColumns));
            
            Optional<SuperCsvRowException> rowException = Optional.empty();
            try {
                executeCellProcessor(processedColumns, getColumns(), beanMappingCache.getCellProcessorsForReading(), context);
                
            } catch(SuperCsvRowException e) {
                /*
                 * カラムごとのCellProcessorのエラーの場合、別なValidatorで値を検証するために、
                 * 後から判定を行うようにする。
                 */
                rowException = Optional.of(e);
                
                final List<CsvError> errors = exceptionConverter.convert(e, beanMappingCache.getOriginal());
                bindingErrors.addAllErrors(errors);
                
            } catch(SuperCsvException e) {
                errorMessages.addAll(exceptionConverter.convertAndFormat(e, beanMappingCache.getOriginal()));
                throw e;
            }
            
            // コールバックメソッドの実行（読み込み前）
            for(CallbackMethod callback : beanMappingCache.getOriginal().getPreReadMethods()) {
                callback.invoke(bean, context, bindingErrors, beanMappingCache.getOriginal());
            }
            
            // beanへのマッピング
            populateBean(bean, beanMappingCache.getNameMapping(), bindingErrors);
            
            // Bean(レコード)の入力値検証
            for(CsvValidator<T> recordValidator : validators) {
                recordValidator.validate(bean, bindingErrors, new ValidationContext<>(context, beanMappingCache.getOriginal()));
            }
            
            // コールバックメソッドの実行（読み込み後）
            for(CallbackMethod callback : beanMappingCache.getOriginal().getPostReadMethods()) {
                callback.invoke(bean, context, bindingErrors, beanMappingCache.getOriginal());
            }
            
            // エラーメッセージの変換
            processErrors(bindingErrors, context, rowException);
            
            return bean;
            
        }
        
        return null; // EOF
        
        
    }
902836
def guess_fileformat(aUri):
    
    if aUri.endswith(".xml"):
        return "xml"
    elif aUri.endswith(".nt"):
        return "nt"
    elif aUri.endswith(".n3") or aUri.endswith(".ttl"):
        return "n3"
    elif aUri.endswith(".trix"):
        return "trix"
    elif aUri.endswith(".rdfa"):
        return "rdfa"
    else:  # to be handled outside this method
        return None