Convenience method which tokenizes the URLs and the SMILEYS_MAPPING, removes accents
and symbols and eliminates the extra spaces from the provided text.

@param text
@return
435855
public static String clear(String text) {
        text = StringCleaner.tokenizeURLs(text);
        text = StringCleaner.tokenizeSmileys(text);
        text = StringCleaner.removeAccents(text);
        text = StringCleaner.removeSymbols(text);
        text = StringCleaner.removeExtraSpaces(text);
        
        return text.toLowerCase(Locale.ENGLISH);
    }
352587
public String senTokenize(String text){
		String ret = text;
		
		if (isTokenization){
			ret = PennTokenizer.tokenize(text);
		}
		
		return ret.trim();
	}
301951
public static String encodeText(String text, Charset charset) {
		try {
			return MimeUtility.encodeText(text, charset.name(), null);
		} catch (UnsupportedEncodingException e) {
			// ignore
		}
		return text;
	}
239602
public static String urlDecode(String text)
    {
        String urlEncodedString=text;
        if(text!=null)
        {
            try
            {
                urlEncodedString=URLDecoder.decode(text,SpiUtil.UTF_8_ENCODING_NAME);
            }
            catch(UnsupportedEncodingException exception)
            {
                throw new FaxException("Error while URL decoding text.",exception);
            }
        }
        
        return urlEncodedString;
    }
226619
public String replace(CharSequence text)
    {
        StringBuilder sb = new StringBuilder();
        replace(sb, text);
        return sb.toString();
    }
718519
@Deprecated
    public static String escapeText(String text) {
        if (text == null) {
            return null;
        }

        StringBuilder sb = SharedStringBuilder.get(SB_ESCAPE);

        for (int i = 0; i < text.length(); i++) {
            char ch = text.charAt(i);
            switch (ch) {
                case '"':
                    sb.append("\\\"");
                    break;
                case '\\':
                    sb.append("\\\\");
                    break;
                case '\b':
                    sb.append("\\b");
                    break;
                case '\f':
                    sb.append("\\f");
                    break;
                case '\n':
                    sb.append("\\n");
                    break;
                case '\r':
                    sb.append("\\r");
                    break;
                case '\t':
                    sb.append("\\t");
                    break;
                case '/':
                    sb.append("\\/");
                    break;
                default:
                    //Reference: http://www.unicode.org/versions/Unicode5.1.0/
                    if ((ch >= '\u0000' && ch <= '\u001F') || (ch >= '\u007F' && ch <= '\u009F') || (ch >= '\u2000' && ch <= '\u20FF')) {
                        String ss = Integer.toHexString(ch);
                        sb.append("\\u");
                        for (int k = 0; k < 4 - ss.length(); k++) {
                            sb.append('0');
                        }
                        sb.append(ss.toUpperCase());
                    }
                    else {
                        sb.append(ch);
                    }
            }
        }

        return sb.toString();
    }
607667
public static String makeLinear (String text)
    {
        if (text == null) {
            return text;
        }
        // handle both line ending formats
        text = text.replace("\n", "<br>\n");
        return text;
    }
946556
@Pure
	public static String toHTML(String text) {
		if (text == null) {
			return null;
		}
		final Map<Character, String> transTbl = getJavaToHTMLTranslationTable();
		assert transTbl != null;
		if (transTbl.isEmpty()) {
			return text;
		}
		final StringBuilder patternStr = new StringBuilder();
		for (final Character c : transTbl.keySet()) {
			if (patternStr.length() > 0) {
				patternStr.append("|"); //$NON-NLS-1$
			}
			patternStr.append(Pattern.quote(c.toString()));
		}
		final Pattern pattern = Pattern.compile(patternStr.toString());
		final Matcher matcher = pattern.matcher(text);
		final StringBuilder result = new StringBuilder();
		String character;
		String entity;
		int lastIndex = 0;
		while (matcher.find()) {
			final int idx = matcher.start();
			result.append(text.substring(lastIndex, idx));
			lastIndex = matcher.end();
			character = matcher.group();
			if (character.length() == 1) {
				entity = transTbl.get(Character.valueOf(character.charAt(0)));
				if (entity != null) {
					entity = "&" + entity + ";"; //$NON-NLS-1$ //$NON-NLS-2$
				} else {
					entity = character;
				}
			} else {
				entity = character;
			}
			result.append(entity);
		}
		if (lastIndex < text.length()) {
			result.append(text.substring(lastIndex));
		}
		return result.toString();
	}
1155094
function(text) {

            urls = [];
            titles = [];
            htmlBlock = [];

            text = text.replace(/~/g,"~T");
            text = text.replace(/\$/g,"~D");
            text = text.replace(/\r\n/g,"\n");
            text = text.replace(/\r/g,"\n");
            text = "\n\n" + text + "\n\n";
            text = detab(text);
            text = text.replace(/^[ \t]+$/mg,"");
            text = hashHTMLBlocks(text);
            text = stripLinkDefinitions(text);
            text = runBlockGamut(text);
            text = unescapeSpecialChars(text);
            text = text.replace(/~D/g,"$$");
            text = text.replace(/~T/g,"~");

            return text;
        }
749311
public static String urlEncode(final String text) {
        try {
            return URLEncoder.encode(text, StandardCharsets.UTF_8.name());
        } catch (final UnsupportedEncodingException e) {
            final String message = "Unable to encode text : " + text;
            throw new TechnicalException(message, e);
        }
    }
813152
public String senSegment(String text){
		String ret = text;
		
		//Segment sentences
		if (vnSenSegmenter != null){
			ret = vnSenSegmenter.senSegment(text);
		}
		
		return ret.trim();
	}
558288
public static byte[] decodeLines(final String text) {
       char[] buf = new char[text.length()];
       int pValue = 0;
       for (int ip = 0; ip < text.length(); ip++) {
           final char cValue = text.charAt(ip);
           if (cValue != ' ' && cValue != '\r' && cValue != '\n' && cValue != '\t') {
               buf[pValue++] = cValue;
           }
       }
       return decode(buf, 0, pValue);
   }
547958
public static String hexHtmlify(String text) {
		String emojifiedStr = emojify(text);
		return htmlifyHelper(emojifiedStr, true, false);
	}
918560
public static String shortCodify(String text) {
		String emojifiedText = emojify(text);

		// TODO - this approach is ugly, need to find an optimal way to replace
		// the emojis
		// could not find an ideal way to identify emojis in the passed string
		// characters like <3 has multiple characters, but doesn't have
		// surrogate pairs
		// so at this point, we iterate through all the emojis and replace with
		// short codes
		for (Emoji emoji : EmojiManager.data()) {
			StringBuilder shortCodeBuilder = new StringBuilder();
			shortCodeBuilder.append(":").append(emoji.getAliases().get(0)).append(":");

			emojifiedText = emojifiedText.replace(emoji.getEmoji(), shortCodeBuilder.toString());
		}
		return emojifiedText;
	}
1296178
private String replaceParameters(String text) {
    	Map<String, Object> params  = bean.getParametersBean().getParamValues();
        for  (String param : params.keySet()) {
             text = StringUtil.replace(text, "\\$P\\{" + param + "\\}", StringUtil.getValueAsString(params.get(param), null));
        }
        return text;
    }
506945
protected String removeInvalidHeaderCommentsAndProcessVelocityMacros(String text) {
        String answer = "";
        String[] lines = text.split("\r?\n");
        for (String line : lines) {
            String l = line.trim();
            // a bit of Velocity here
            if (!l.startsWith("##") && !l.startsWith("#set(")) {
                if (line.contains("${D}")) {
                    line = line.replaceAll("\\$\\{D\\}", "\\$");
                }
                answer = answer.concat(line);
                answer = answer.concat(System.lineSeparator());
            }
        }
        return answer;
    }
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }