convert iso character coding to unicode (PHP conform)
needed for TTF text generation of special characters (Latin-2)

@access private
960342
function _iso2uni($isoline)
    {
        $iso2uni = array(
            173 => "&#161;",
            155 => "&#162;",
            156 => "&#163;",
            15 => "&#164;",
            157 => "&#165;",
            124 => "&#166;",
            21 => "&#167;",
            249 => "&#168;",
            184 => "&#169;",
            166 => "&#170;",
            174 => "&#171;",
            170 => "&#172;",
            169 => "&#174;",
            238 => "&#175;",
            248 => "&#176;",
            241 => "&#177;",
            253 => "&#178;",
            252 => "&#179;",
            239 => "&#180;",
            230 => "&#181;",
            20 => "&#182;",
            250 => "&#183;",
            247 => "&#184;",
            251 => "&#185;",
            167 => "&#186;",
            175 => "&#187;",
            172 => "&#188;",
            171 => "&#189;",
            243 => "&#190;",
            168 => "&#191;",
            183 => "&#192;",
            181 => "&#193;",
            182 => "&#194;",
            199 => "&#195;",
            142 => "&#196;",
            143 => "&#197;",
            146 => "&#198;",
            128 => "&#199;",
            212 => "&#200;",
            144 => "&#201;",
            210 => "&#202;",
            211 => "&#203;",
            141 => "&#204;",
            161 => "&#205;",
            140 => "&#206;",
            139 => "&#207;",
            209 => "&#208;",
            165 => "&#209;",
            227 => "&#210;",
            224 => "&#211;",
            226 => "&#212;",
            229 => "&#213;",
            153 => "&#214;",
            158 => "&#215;",
            157 => "&#216;",
            235 => "&#217;",
            233 => "&#218;",
            234 => "&#219;",
            154 => "&#220;",
            237 => "&#221;",
            232 => "&#222;",
            225 => "&#223;",
            133 => "&#224;",
            160 => "&#225;",
            131 => "&#226;",
            198 => "&#227;",
            132 => "&#228;",
            134 => "&#229;",
            145 => "&#230;",
            135 => "&#231;",
            138 => "&#232;",
            130 => "&#233;",
            136 => "&#234;",
            137 => "&#235;",
            141 => "&#236;",
            161 => "&#237;",
            140 => "&#238;",
            139 => "&#239;",
            208 => "&#240;",
            164 => "&#241;",
            149 => "&#242;",
            162 => "&#243;",
            147 => "&#244;",
            228 => "&#245;",
            148 => "&#246;",
            246 => "&#247;",
            155 => "&#248;",
            151 => "&#249;",
            163 => "&#250;",
            150 => "&#251;",
            129 => "&#252;",
            236 => "&#253;",
            231 => "&#254;",
            152 => "&#255;"
        );
        for ($i = 0; $i < strlen($isoline); $i++)
        {
            $thischar = substr($isoline, $i, 1);
            $new = $iso2uni[ord($thischar)];
            $uniline .= ($new != "") ? $new : $thischar;
        }
        return $uniline;
    }
113737
public function toAscii()
    {
        if ($this->isIdn() && $this->isUnicode())
            $this->host = (new IdnaConvert())->encode($this->host);
    }
107125
def unicode2encode(text, charmap):
    '''
    
    '''
    if isinstance(text, (list, tuple)):
        unitxt = ''
        for line in text:
            for val,key in charmap.items():
                if key in line:
                    line = line.replace(key, val)
                # end of if val in text:
            unitxt += line
        # end of for line in text:
        return unitxt
    elif isinstance(text, str):
        for val,key in charmap.items():
            if key in text:
                text = text.replace(key, val)
        return text
1071992
static function &charsetList()
    {
        $charsets =& $GLOBALS["eZMBCharsetList"];
        if ( !is_array( $charsets ) )
        {
            $charsetList = array( "ucs-4", "ucs-4be", "ucs-4le", "ucs-2", "ucs-2be", "ucs-2le", "utf-32", "utf-32be", "utf-32le", "utf-16",
                                  "utf-16be", "utf-16le", "utf-8", "utf-7", "ascii", "euc-jp", "sjis", "eucjp-win", "sjis-win", "iso-2022-jp", "jis",
                                  "iso-8859-1", "iso-8859-2", "iso-8859-3", "iso-8859-4", "iso-8859-5", "iso-8859-6", "iso-8859-7", "iso-8859-8",
                                  "iso-8859-9", "iso-8859-10", "iso-8859-13", "iso-8859-14", "iso-8859-15", "byte2be", "byte2le", "byte4be",
                                  "byte4le", "base64", "7bit", "8bit", "utf7-imap" );
            $charsets = array();
            foreach ( $charsetList as $charset )
            {
                $charsets[$charset] = $charset;
            }
        }
        return $charsets;
    }
676340
public function convertToUtf8($str, $charset = 'ISO-8859-1')
    {
        if ($charset == 'default') {
            $charset = 'utf-8';
        }

        $str = iconv($charset, 'UTF-8//IGNORE', $str);

        return $str;
    }
529105
function utf8Encode(str) {
  var bytes = [], offset = 0, length, char;

  str = encodeURI(str);
  length = str.length;

  while (offset < length) {
    char = str.charAt(offset);
    offset += 1;

    if (char !== '%') {
      bytes.push(char.charCodeAt(0));
    } else {
      char = str.charAt(offset) + str.charAt(offset + 1);
      bytes.push(parseInt(char, 16));
      offset += 2;
    }
  }

  return bytes;
}
1333308
function encodeToUTF8($str)
    {
        $encodage = mb_detect_encoding($str, "UTF-8, ISO-8859-1, ISO-8859-15, windows-1252", true);
        $str_utf8 = mb_convert_encoding($str, "UTF-8", $encodage);
        return $str_utf8;
    }
1002853
public function getFormatEncodingOptions()
    {
        $options = [
            'utf-8',
            'us-ascii',
            'iso-8859-1',
            'iso-8859-2',
            'iso-8859-3',
            'iso-8859-4',
            'iso-8859-5',
            'iso-8859-6',
            'iso-8859-7',
            'iso-8859-8',
            'iso-8859-0',
            'iso-8859-10',
            'iso-8859-11',
            'iso-8859-13',
            'iso-8859-14',
            'iso-8859-15',
            'Windows-1251',
            'Windows-1252'
        ];

        $translated = array_map(function ($option) {
            return Lang::get('backend::lang.import_export.encodings.'.Str::slug($option, '_'));
        }, $options);

        return array_combine($options, $translated);
    }
254544
function convert_text($text)
    {
        global $debugObject;
        if (is_object($debugObject)) {$debugObject->debugLogEntry(1);}

        $converted_text = $text;

        $sourceCharset = "";
        $targetCharset = "";

        if ($this->dom)
        {
            $sourceCharset = strtoupper($this->dom->_charset);
            $targetCharset = strtoupper($this->dom->_target_charset);
        }
        if (is_object($debugObject)) {$debugObject->debugLog(3, "source charset: " . $sourceCharset . " target charaset: " . $targetCharset);}

        if (!empty($sourceCharset) && !empty($targetCharset) && (strcasecmp($sourceCharset, $targetCharset) != 0))
        {
            // Check if the reported encoding could have been incorrect and the text is actually already UTF-8
            if ((strcasecmp($targetCharset, 'UTF-8') == 0) && ($this->is_utf8($text)))
            {
                $converted_text = $text;
            }
            else
            {
                $converted_text = iconv($sourceCharset, $targetCharset, $text);
            }
        }

        // Lets make sure that we don't have that silly BOM issue with any of the utf-8 text we output.
        if ($targetCharset == 'UTF-8')
        {
            if (substr($converted_text, 0, 3) == "\xef\xbb\xbf")
            {
                $converted_text = substr($converted_text, 3);
            }
            if (substr($converted_text, -3) == "\xef\xbb\xbf")
            {
                $converted_text = substr($converted_text, 0, -3);
            }
        }

        return $converted_text;
    }
652793
private function _out_string_encoding( &$str )
     {
        $rsc = $this->_source_result_charset();
        if( $rsc==1 ) {
            $rsstr = iconv(UCS2, 'utf-8', $str);
        }
        else if( $rsc==2 ) {
            $rsstr = iconv('utf-8', 'gb18030', iconv(UCS2, 'utf-8', $str) );
        }
        else{
            $rsstr = iconv('utf-8', 'big5', iconv(UCS2, 'utf-8', $str) );
        }
        return $rsstr;
     }
1183414
function __encode_utf8(text) {

        // Fill buffer with UTF-8
        for (var i=0; i<text.length; i++) {
            var codepoint = text.charCodeAt(i);
            __append_utf8(codepoint);
        }

        // Flush buffer
        if (length > 0) {
            var out_buffer = buffer.subarray(0, length);
            length = 0;
            return out_buffer;
        }

    }
1120701
function isUTF8()
    {
        $string = (string) $this;
        if (function_exists('utf8_decode'))
            $decodedStr = strlen(utf8_decode($string));
        elseif (function_exists('mb_convert_encoding'))
            $decodedStr = strlen(utf8_decode($string));
        else
            $decodedStr = $string;


        return strlen($string) != strlen($decodedStr);
    }
486279
function encode(text, lang, encoding) {

    // ISO/IANA language code, but we're not enforcing
    if (!lang) { lang = 'en'; }

    var encoded = util.stringToBytes(lang + text);
    encoded.unshift(lang.length);

    return encoded;
}
1258773
function process($text,$options = array()){
		$text = (string)$text;
		$options = $this->_getOptions($options);

		$attrs = $options["attrs"];
		$mailto_attrs = $options["mailto_attrs"];
		$utf8 = $options["utf8"] ? "u" : "";

		$tr_table = $this->_prepareTextTrTable($text,$options);
		$tr_table_rev = sizeof($tr_table)>0 ? array_combine(array_values($tr_table),array_keys($tr_table)) : array(); // in PHP5.3 parameters of array_combine should have at least 1 element

		if($options["escape_html_entities"]){
			$flags =  ENT_COMPAT;
			if(defined("ENT_HTML401")){ $flags = $flags | ENT_HTML401; }

			// As of PHP5.4 the default encoding is UTF-8, it causes troubles in non UTF-8 applications.
			// It seems that the encoding ISO-8859-1 works well in UTF-8 applications.
			$text = htmlspecialchars($text,$flags,"ISO-8859-1");
		}
		$text = strtr($text,$tr_table);

		$this->__attrs = $attrs;
		$this->__mailto_attrs = $mailto_attrs;
		$this->__options = $options;
		$this->__replaces = array();

		// Data for patterns
		$url_allowed_chars = "[-a-zA-Z0-9@:%_+.~#?&\\/=;\[\]]"; // According to https://stackoverflow.com/questions/1547899/which-characters-make-a-url-invalid/1547940#1547940 there are yet more characters: !$'()*`,
		$domain_name_part = "[a-zA-Z0-9][-a-zA-Z0-9]+"; // without dot
		$optional_port = "(:[1-9][0-9]{1,4}|)"; // ":81", ":65535", ""
		$top_level_domains = "(".join("|",$this->top_level_domains).")";
		$username_chars = "[-a-zA-Z0-9%]+";
		$password_chars = $username_chars;

		// urls starting with http://, http://, ftp:/ and containing username and password
		$text = preg_replace_callback("/(?<first_char>.?)\b(?<link>(ftp|https?):\\/\\/$username_chars:$password_chars@$domain_name_part(\.$domain_name_part)*$optional_port(\/$url_allowed_chars*|))/i$utf8",array($this,"_replaceLink"),$text);

		// urls starting with http://, http://, ftp:/
		$text = preg_replace_callback("/(?<first_char>.?)\b(?<link>(ftp|https?):\\/\\/$domain_name_part(\.$domain_name_part)*$optional_port(\/$url_allowed_chars*|))/i$utf8",array($this,"_replaceLink"),$text);

		// urls starting with www.
		$text = preg_replace_callback("/(?<first_char>.?)\b(?<link>www\.$domain_name_part(\.$domain_name_part)*$optional_port(\/$url_allowed_chars*|))/i$utf8",array($this,"_replaceLink"),$text);

		// emails
		$text = preg_replace_callback("/(?<address>[_.0-9a-z-]+@([0-9a-z][0-9a-z-]+\\.)+[a-z]{2,5})(?<ending_interrupter>.?)/i$utf8",array($this,"_replaceEmail"),$text);

		// urls without leading www., http://, ...
		$text = preg_replace_callback("/(?<first_char>.?)\b(?<link>($domain_name_part\\.)+$top_level_domains$optional_port\b(\/$url_allowed_chars*|))/i$utf8",array($this,"_replaceLink"),$text);

		$text = strtr($text,$this->__replaces);

		unset($this->__attrs);
		unset($this->__mailto_attrs);
		unset($this->__options);
		unset($this->__replaces);

		$text = strtr($text,$tr_table_rev);

		return $text;
	}
1338023
private static function unicodeToEntitiesPreservingAscii($unicode)
    {
        $entities = '';

        foreach ($unicode as $value) {
            if ($value != 65279) {
                $entities .= $value > 127
                             ? '\uc0{\u' . $value . '}'
                             : chr($value);
            }
        }

        return $entities;
    }
900595
public function process($value, $preserve_keys = false)
    {
        if (is_array($value)) {
            return array_reverse($value, $preserve_keys);
        } elseif (($charset = $this->core->getCharset()) === 'iso-8859-1') {
            return strrev((string)$value);
        }

        $strlen = mb_strlen($value);
        $out    = '';
        while ($strlen --) {
            $out .= mb_substr($value, $strlen, 1, $charset);
        }

        return $out;
    }
827086
def encode(text, orig_coding):
    
    if orig_coding == 'utf-8-bom':
        return BOM_UTF8 + text.encode("utf-8"), 'utf-8-bom'

    # Try declared coding spec
    coding = get_coding(text)
    if coding:
        try:
            return text.encode(coding), coding
        except (UnicodeError, LookupError):
            raise RuntimeError("Incorrect encoding (%s)" % coding)
    if (orig_coding and orig_coding.endswith('-default') or
       orig_coding.endswith('-guessed')):
        coding = orig_coding.replace("-default", "")
        coding = orig_coding.replace("-guessed", "")
        try:
            return text.encode(coding), coding
        except (UnicodeError, LookupError):
            pass

    # Try saving as ASCII
    try:
        return text.encode('ascii'), 'ascii'
    except UnicodeError:
        pass

    # Save as UTF-8 without BOM
    return text.encode('utf-8'), 'utf-8'