/*
FIX:remove if any special characters exists in keywords
330770
public static String getcompressedKeyword(Text keyword)
	{
		String compressedKeyword = "";
		String key = keyword.getText();
		String remove = " ()+,/:[]<>\"*";
		for (int i = 0; i < key.length(); i++)
		{
			if (remove.indexOf(key.charAt(i)) == -1)
			{
				compressedKeyword = compressedKeyword + key.charAt(i);
			}
		}

		return compressedKeyword.toUpperCase();
	}
1079283
function fixUnicode(txt) {
    txt = txt.replace(/\003/g, ' ');
    txt = txt.replace(/\004/g, ' ');
    txt = txt.replace(/(\r|\n)/g, ' ');
    txt = txt.replace(/[\u007f-\uffff]/g, function (c) {
        return '\\u' + ('0000' + c.charCodeAt(0).toString(16)).slice(-4);
    });
    return txt;
}
1283527
private void stripAsterisk(final StringBuilder javaDoc) {
		int index = javaDoc.indexOf("*");
		while (index != -1) {
			javaDoc.replace(index, index + 1, "");
			index = javaDoc.indexOf("*");
		}
	}
829596
function unescapeSpecialChars(text) {

        text = text.replace(/~E(\d+)E/g, function(wholeMatch,m1) {
            var charCodeToReplace = parseInt(m1);
            return String.fromCharCode(charCodeToReplace);
        });

        return text;
    }
1004924
private function prepareKeywords($keywords)
    {
        if ($keywords === null)
            return null;

        if (is_array($keywords))
            $keywords = implode(', ', $keywords);

        return strtolower(strip_tags($keywords));
    }
276109
function fixedEncodeURIComponent (str) {
    return encodeURIComponent(str).replace(/[!'()*]/g, function(c) {
      return '%' + c.charCodeAt(0).toString(16);
    });
  }
565205
def clean_text(s):
    """"""
    SPACES_RE = re.compile(r'\s+')
    SPECIAL_CHARS_RE = re.compile(r'[^\w\s\.\-\(\)]')
    s = SPACES_RE.sub(' ', s)
    s = s.strip()
    s = SPECIAL_CHARS_RE.sub('', s)
    return s
270495
def clean_title title
      title.gsub(GENDER_FILTER,'').
            gsub(SYMBOL_FILTER,'').
            gsub(WORDS_IN_BRACKETS, '\1').
            gsub(CODE_TOKEN_FILTER,'').
            gsub(WHITESPACE,' ').
            downcase.
            strip
    end
1184545
function replaceSensitive(chr, dbl) {

		var result,
		    dbl_result;

		if (typeof baseDiacriticsMap[chr] === 'undefined') {
			return chr;
		}

		result = '[' + chr
		result += baseDiacriticsMap[chr];
		result += ']';

		if (dbl && baseDiacriticsMap[dbl]) {
			dbl_result = baseDiacriticsMap[dbl];

			// Make the previous group optional,
			// but do require one of these groups then
			result = '?(?:' + result + '|[' + dbl_result + '])';
		}

		return result;
	}
551663
function processKeywords (text) {
  // for some reason retext is not filtering "it's" out
  const cleanText = text.replace(/'t/g, '')
                        .replace(/'s/g, '')
                        .replace(/'ll/g, '')
                        .replace(/'re/g, '')
                        .replace(/'ve/g, '');

  return new Promise(function (resolve, reject) {
    processor.process(cleanText, function (err, file) {
      if (err) {
        reject(err);
      }
      const space = file.namespace('retext');
      const words = space.keywords.map((w) => ({
        value: nlcstToString(w.matches[0].node),
        score: w.score
      }));
      const phrases = space.keyphrases.map((p) => ({
        value: p.matches[0].nodes.map(nlcstToString).join(''),
        score: p.score
      }));

      resolve({words, phrases});
    });
  });
}
316072
function optimize_keyword_group($key) {
        $this->language_data['CACHED_KEYWORD_LISTS'][$key] =
            $this->optimize_regexp_list($this->language_data['KEYWORDS'][$key]);
        $space_as_whitespace = false;
        if(isset($this->language_data['PARSER_CONTROL'])) {
            if(isset($this->language_data['PARSER_CONTROL']['KEYWORDS'])) {
                if(isset($this->language_data['PARSER_CONTROL']['KEYWORDS']['SPACE_AS_WHITESPACE'])) {
                    $space_as_whitespace = $this->language_data['PARSER_CONTROL']['KEYWORDS']['SPACE_AS_WHITESPACE'];
                }
                if(isset($this->language_data['PARSER_CONTROL']['KEYWORDS'][$key]['SPACE_AS_WHITESPACE'])) {
                    if(isset($this->language_data['PARSER_CONTROL']['KEYWORDS'][$key]['SPACE_AS_WHITESPACE'])) {
                        $space_as_whitespace = $this->language_data['PARSER_CONTROL']['KEYWORDS'][$key]['SPACE_AS_WHITESPACE'];
                    }
                }
            }
        }
        if($space_as_whitespace) {
            foreach($this->language_data['CACHED_KEYWORD_LISTS'][$key] as $rxk => $rxv) {
                $this->language_data['CACHED_KEYWORD_LISTS'][$key][$rxk] =
                    str_replace(" ", "\\s+", $rxv);
            }
        }
    }
497689
function Parser (customKeywordSpec) {
    var keywordSpec = customKeywordSpec || {
        _: [0],
        gettext: [0],
        ngettext: [0, 1]
    };
    var openings = ['<%', '{\\*', '{{'];
    var closures = ['%>', '\\*}', '}}'];

    if (typeof keywordSpec !== 'object') {
        throw 'Invalid keyword spec';
    }

    this.keywordSpec = keywordSpec;
    this.expressionPattern = new RegExp([
        '(?:' + openings.join('[=-]|') + '[=-]) *',
        '(' + Object.keys(keywordSpec).map(escapeRegExp).join('|') + ')',
        '\\(',
        '([\\s\\S]*?)',
        '\\)',
        ' *(?:' + closures.join('|') + ')'
    ].join(''), 'g');
}
1134709
function removeUnwantedCharacters(cleanedString, unwantedCharArray) {
    for (var x = 0; x < unwantedCharArray.length; x++) {
        var unwantedCharRegExp = new RegExp(unwantedCharArray[x], 'g');
        cleanedString = cleanedString.replace(unwantedCharRegExp, '');
    }
    return cleanedString;
}
977693
function escapeSpecial(value) {
    let specialMaps = {
        '&lt;': '<',
        '&gt;': '>',
        '&quote;': '"',
        '&#39;': '\''
    };
    for (let n in specialMaps) {
        value = value.replace(new RegExp(n, 'g'), specialMaps[n]);
    }
    return value;
}
631945
function normalizeValue(value) {
		if (value.charAt(0) == '-' && !/^\-[\.\d]/.test(value)) {
			value = value.replace(/^\-+/, '');
		}
		
		var ch = value.charAt(0);
		if (ch == '#') {
			return normalizeHexColor(value);
		}

		if (ch == '$') {
			return utils.escapeText(value);
		}

		return getKeyword(value);
	}
644582
private function handle_keyword_replace($match)
    {
        $k = $this->_kw_replace_group;
        $keyword = $match[0];
        $keyword_match = $match[1];

        $before = '';
        $after = '';

        if ($this->keyword_links) {
            // Keyword links have been ebabled

            if (isset($this->language_data['URLS'][$k]) &&
                $this->language_data['URLS'][$k] != ''
            ) {
                // There is a base group for this keyword

                // Old system: strtolower
                //$keyword = ( $this->language_data['CASE_SENSITIVE'][$group] ) ? $keyword : strtolower($keyword);
                // New system: get keyword from language file to get correct case
                if (!$this->language_data['CASE_SENSITIVE'][$k] &&
                    strpos($this->language_data['URLS'][$k], '{FNAME}') !== false
                ) {
                    foreach ($this->language_data['KEYWORDS'][$k] as $word) {
                        if (strcasecmp($word, $keyword_match) == 0) {
                            break;
                        }
                    }
                }
                else {
                    $word = $keyword_match;
                }

                $before = '<|UR1|"' .
                          str_replace(
                              [
                                  '{FNAME}',
                                  '{FNAMEL}',
                                  '{FNAMEU}',
                                  '.'],
                              [
                                  str_replace('+', '%20', urlencode($this->hsc($word))),
                                  str_replace('+', '%20', urlencode($this->hsc(strtolower($word)))),
                                  str_replace('+', '%20', urlencode($this->hsc(strtoupper($word)))),
                                  '<DOT>'],
                              $this->language_data['URLS'][$k]
                          ) . '">';
                $after = '</a>';
            }
        }

        return $before . '<|/' . $k . '/>' . $this->change_case($keyword) . '|>' . $after;
    }
425400
function sanitize(lang, filter) {
  if(!(typeof lang == 'string')) return lang;
  lang = lang.replace(/\..*$/, '').toLowerCase();
  if(typeof filter == 'function') {
    lang = filter(lang);
  }
  return lang;
}