编码转换函数,对整个文件进行编码转换
支持以下转换
GB2312、UTF-8 WITH BOM转换为UTF-8
UTF-8、UTF-8 WITH BOM转换为GB2312
@access public
@param string $filename		文件名
@param string $out_charset	转换后的文件编码,与iconv使用的参数一致
@return void
180435
static function DetectAndSwitch($filename,$out_charset) {
		$fpr = fopen($filename,"r");
		$char1 = fread($fpr,1);
		$char2 = fread($fpr,1);
		$char3 = fread($fpr,1);

		$originEncoding = "";

		if($char1==chr(239) && $char2==chr(187) && $char3==chr(191))//UTF-8 WITH BOM
			$originEncoding = "UTF-8 WITH BOM";
		elseif($char1==chr(255) && $char2==chr(254))//UNICODE LE
		{
			self::error("不支持从UNICODE LE转换到UTF-8或GB编码");
			fclose($fpr);
			return;
		}elseif($char1==chr(254) && $char2==chr(255)){//UNICODE BE
			self::error("不支持从UNICODE BE转换到UTF-8或GB编码");
			fclose($fpr);
			return;
		}else{//没有文件头,可能是GB或UTF-8
			if(rewind($fpr)===false){//回到文件开始部分,准备逐字节读取判断编码
				self::error($filename."文件指针后移失败");
				fclose($fpr);
				return;
			}

			while(!feof($fpr)){
				$char = fread($fpr,1);
				//对于英文,GB和UTF-8都是单字节的ASCII码小于128的值
				if(ord($char)<128)
					continue;

				//对于汉字GB编码第一个字节是110*****第二个字节是10******(有特例,比如联字)
				//UTF-8编码第一个字节是1110****第二个字节是10******第三个字节是10******
				//按位与出来结果要跟上面非星号相同,所以应该先判断UTF-8
				//因为使用GB的掩码按位与,UTF-8的111得出来的也是110,所以要先判断UTF-8
				if((ord($char)&224)==224) {
					//第一个字节判断通过
					$char = fread($fpr,1);
					if((ord($char)&128)==128) {
						//第二个字节判断通过
						$char = fread($fpr,1);
						if((ord($char)&128)==128) {
							$originEncoding = "UTF-8";
							break;
						}
					}
				}
				if((ord($char)&192)==192) {
					//第一个字节判断通过
					$char = fread($fpr,1);
					if((ord($char)&128)==128) {
						//第二个字节判断通过
						$originEncoding = "GB2312";
						break;
					}
				}
			}
		}

		if(strtoupper($out_charset)==$originEncoding) {
			self::info("文件".$filename."转码检查完成,原始文件编码".$originEncoding);
			fclose($fpr);
		}else {
			//文件需要转码
			$originContent = "";

			if($originEncoding == "UTF-8 WITH BOM") {
				//跳过三个字节,把后面的内容复制一遍得到utf-8的内容
				fseek($fpr,3);
				$originContent = fread($fpr,filesize($filename)-3);
				fclose($fpr);
			}elseif(rewind($fpr)!=false){//不管是UTF-8还是GB2312,回到文件开始部分,读取内容
				$originContent = fread($fpr,filesize($filename));
				fclose($fpr);
			}else{
				self::error("文件编码不正确或指针后移失败");
				fclose($fpr);
				return;
			}

			//转码并保存文件
			$content = iconv(str_replace(" WITH BOM","",$originEncoding),strtoupper($out_charset),$originContent);
			$fpw = fopen($filename,"w");
			fwrite($fpw,$content);
			fclose($fpw);

			if($originEncoding!="")
				self::info("对文件".$filename."转码完成,原始文件编码".$originEncoding.",转换后文件编码".strtoupper($out_charset));
			elseif($originEncoding=="")
				self::info("文件".$filename."中没有出现中文,但是可以断定不是带BOM的UTF-8编码,没有进行编码转换,不影响使用");
		}
	}
110039
public function setInputCharset($charset)
    {
        $flag = true;
        
        $charset  = strtolower($charset);
        $charsets = array('utf-8', 'windows-1256', 'cp1256', 'iso-8859-6');
        
        if (in_array($charset, $charsets)) {
            if ($charset == 'windows-1256') {
                $charset = 'cp1256';
            }
            $this->_inputCharset = $charset;
        } else {
            $flag = false;
        }
        
        return $flag;
    }
957834
protected void send(OutputStream out, Charset charset) throws IOException {
        // 报文分为三个部分
        // 报文头
        byte[] head = request.getHeadByte(charset);
        // 交易参数
        byte[] param = request.encodeParam(charset);
        // 交易文件流
        InputStream inputFile = request.getInputFile();
        long fileSize = request.getFileSize();
        LOGGER.debug("发出交易请求..报文头为{}", request.getHead());
        LOGGER.debug("交易参数为{}", param);
        // 输出报文头
        out.write(head);
        // 输出交易参数
        if (null != param) {
            out.write(param);
        }
        // 输出文件流
        if (null != inputFile) {
            sendFileContent(inputFile, fileSize, out);
        }
    }
1217866
public static function utf7_to_utf8($str)
    {
        $Index_64 = array(
            0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
            0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
            0,0,0,0, 0,0,0,0, 0,0,0,1, 0,0,0,0,
            1,1,1,1, 1,1,1,1, 1,1,0,0, 0,0,0,0,
            0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,
            1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,0,
            0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,
            1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,0,
        );

        $u7len = strlen($str);
        $str   = strval($str);
        $res   = '';

        for ($i=0; $u7len > 0; $i++, $u7len--) {
            $u7 = $str[$i];
            if ($u7 == '+') {
                $i++;
                $u7len--;
                $ch = '';

                for (; $u7len > 0; $i++, $u7len--) {
                    $u7 = $str[$i];

                    if (!$Index_64[ord($u7)]) {
                        break;
                    }

                    $ch .= $u7;
                }

                if ($ch == '') {
                    if ($u7 == '-') {
                        $res .= '+';
                    }

                    continue;
                }

                $res .= self::utf16_to_utf8(base64_decode($ch));
            }
            else {
                $res .= $u7;
            }
        }

        return $res;
    }
651387
public function save($filename = null, $writeBOM = false)
    {
        if ($filename === null) {
            $filename = $this->filename;
        }

        if (trim($this->fileContent) == '') {
            $this->build();
        }

        $file_content = $this->fileContent;
        if (strtolower($this->encoding) != 'utf-8') {
            if ($this->useIconv) {
                $file_content = iconv('UTF-8', $this->encoding, $file_content);
            } else {
                $file_content = mb_convert_encoding($file_content, $this->encoding, 'UTF-8');
            }
        }

        if ($writeBOM) {
            $file_content = "\xef\xbb\xbf".$file_content;
        }

        $res = file_put_contents($filename, $file_content);
        if (!$res) {
            throw new \Exception('Unable to save the file.');
        }
    }
652793
private function _out_string_encoding( &$str )
     {
        $rsc = $this->_source_result_charset();
        if( $rsc==1 ) {
            $rsstr = iconv(UCS2, 'utf-8', $str);
        }
        else if( $rsc==2 ) {
            $rsstr = iconv('utf-8', 'gb18030', iconv(UCS2, 'utf-8', $str) );
        }
        else{
            $rsstr = iconv('utf-8', 'big5', iconv(UCS2, 'utf-8', $str) );
        }
        return $rsstr;
     }
911166
static function toUTF8( $char_code )
    {
        switch ( $char_code )
        {
            case 0:
                $char = chr( 0 );
            case !($char_code & 0xffffff80): // 7 bit
                $char = chr( $char_code );
                break;
            case !($char_code & 0xfffff800): // 11 bit
                $char = ( chr(0xc0 | (($char_code >> 6) & 0x1f)) .
                          chr(0x80 | ($char_code & 0x3f)) );
                break;
            case !($char_code & 0xffff0000): // 16 bit
                $char = ( chr(0xe0 | (($char_code >> 12) & 0x0f)) .
                          chr(0x80 | (($char_code >> 6) & 0x3f)) .
                          chr(0x80 | ($char_code & 0x3f)) );
                break;
            case !($char_code & 0xffe00000): // 21 bit
                $char = ( chr(0xf0 | (($char_code >> 18) & 0x07)) .
                          chr(0x80 | (($char_code >> 12) & 0x3f)) .
                          chr(0x80 | (($char_code >> 6) & 0x3f)) .
                          chr(0x80 | ($char_code & 0x3f)) );
                break;
            case !($char_code & 0xfc000000): // 26 bit
                $char = ( chr(0xf8 | (($char_code >> 24) & 0x03)) .
                          chr(0x80 | (($char_code >> 18) & 0x3f)) .
                          chr(0x80 | (($char_code >> 12) & 0x3f)) .
                          chr(0x80 | (($char_code >> 6) & 0x3f)) .
                          chr(0x80 | ($char_code & 0x3f)) );
            default: // 31 bit
                $char = ( chr(0xfc | (($char_code >> 30) & 0x01)) .
                          chr(0x80 | (($char_code >> 24) & 0x3f)) .
                          chr(0x80 | (($char_code >> 18) & 0x3f)) .
                          chr(0x80 | (($char_code >> 12) & 0x3f)) .
                          chr(0x80 | (($char_code >> 6) & 0x3f)) .
                          chr(0x80 | ($char_code & 0x3f)) );
        }
        return $char;
    }
594531
static function outputCharsetToPhpCharset($in)
    {
        $out = $in;
        switch ($in) {
            case "None":$out = false;
                break;
            case "UTF8":$out = "UTF-8";
                break;
            case "ISO-8859-1":
            case "ISO-8859-2":
            case "ISO-8859-3":
            case "ISO-8859-4":
            case "ISO-8859-5":
            case "ISO-8859-6":
            case "ISO-8859-7":
            case "ISO-8859-8":
            case "ISO-8859-8-I":
            case "ISO-8859-9":
            case "ISO-8859-10":
            case "ISO-8859-11":
            case "ISO-8859-12":
            case "ISO-8859-13":
            case "ISO-8859-14":
            case "ISO-8859-15":
            case "ISO-8859-16":$out = $in;
                break;
            case "GB18030":
            case "GB2312":
            case "GBK":
            case "Big5":
            case "BIG5":$out = strtolower($in);
                break;
            case "Windows-1250":$out = "Windows-1250";
                break;
            case "Windows-1251":$out = "Windows-1251";
                break;
            case "Windows-1252":$out = "Windows-1252";
                break;
            case "Windows-1253":$out = "Windows-1253";
                break;
            case "Windows-1254":$out = "Windows-1254";
                break;
            case "Windows-1255":$out = "Windows-1255";
                break;
            case "Windows-1256":$out = "Windows-1256";
                break;
            case "Windows-1257":$out = "Windows-1257";
                break;
            case "Windows-1258":$out = "Windows-1258";
                break;
            case "Shift_JIS":$out = "shift-jis";
                break;
            case "TIS-620":$out = "tis-620";
                break;
            default:$out = false;
        }

        return $out;
    }
151839
@Override
  @SuppressForbidden("legitimate sysstreams.")
  public void setOuter(JUnit4 task) {
    this.seed = task.getSeed();

    if (outputFile != null) {
      try {
        Files.createParentDirs(outputFile);
        final CharSink charSink;
        if (append) {
          charSink = Files.asCharSink(outputFile, Charsets.UTF_8, FileWriteMode.APPEND);
        } else {
          charSink = Files.asCharSink(outputFile, Charsets.UTF_8);
        }
        this.output = charSink.openBufferedStream();
      } catch (IOException e) {
        throw new BuildException(e);
      }
    } else {
      if (!UNICODE_ENCODINGS.contains(Charset.defaultCharset().name())) {
        task.log("Your default console's encoding may not display certain" +
            " unicode glyphs: " + Charset.defaultCharset().name(), 
            Project.MSG_INFO);
      }
      output = new LineFlushingWriter(new OutputStreamWriter(System.out, Charset.defaultCharset())) {
        @Override
        // Don't close the underlying stream, just flush.
        public void close() throws IOException {
          flush();
        }
      };
    }
  }
406169
void loadScript(Charset charset) throws IOException {
		this.charset = charset;
		StringBuilder sb = new StringBuilder();
		try (BufferedReader br = Files.newBufferedReader(file.toPath(), charset)) {
			int len;
			char[] buf = new char[1024];
			while ((len = br.read(buf)) != -1) {
				sb.append(buf, 0, len);
			}
		}
		setContents(sb.toString());
		setChanged(false);
		this.lastModified = file.lastModified();
	}
355844
function characet($data, $targetCharset) {
		
		if (!empty($data)) {
			$fileType = $this->fileCharset;
			if (strcasecmp($fileType, $targetCharset) != 0) {
				$data = mb_convert_encoding($data, $targetCharset, $fileType);
				//				$data = iconv($fileType, $targetCharset.'//IGNORE', $data);
			}
		}


		return $data;
	}
653058
function parseTransformationFile( $filename, $name )
    {
//         eZDebug::writeDebug( "Parsing file $filename" );
        $tbl = array();

        $fd = fopen( $filename, "rb" );
        if ( !$fd )
        {
            $this->error( "Failed opening $filename" );
            return false;
        }

        $this->TransformationFiles[] = $name;

        $this->ISOUnicodeCodec = eZTextCodec::instance( 'iso-8859-1', 'unicode' );

        $buffer = '';
        $lineNum = 1;
        $i = 0;
        $hexValues = "0123456789abcdefABCDEF";
        $identifier = false;

        // The big funky parser starts here
        // It starts by reading a chunk of data from the file
        // then splits everything into an array with lines.
        // Then it traverses one line at a time looking for
        // identifiers and rules. Comments will be removed before the
        // line is parsed for identifiers and rules.

        while ( !feof( $fd ) or strlen( $buffer ) > 0 )
        {
            $lines = array();
            $len = strlen( $buffer );
            // Check if we have data in the buffer yet
            // Note: The actual buffer reading is done at the end of this while loop
            if ( $len > 0 )
            {
                $endPos = false;
                $eolPos = 0;
                // Look for complete lines and append to $lines
                while ( $eolPos !== false and $eolPos < $len )
                {
                    $eolPos = strpos( $buffer, "\n", $endPos );
                    if ( $eolPos !== false )
                    {
                        $line = substr( $buffer, $endPos, $eolPos - $endPos );
                        $lines[] = array( 'text' => $line,
                                          'line' => $lineNum );
                        ++$lineNum;
                        $endPos = $eolPos + 1;
                    }
                }

                // If we have leftover data place that back in $buffer
                if ( $endPos !== false )
                {
                    $buffer = substr( $buffer, $endPos );
                }
            }

            // Once we have some lines start parsing them one at a time
            foreach ( $lines as $lineData )
            {
                $line = $lineData['text'];
                $lineOrg = $line;
                $linePos = $lineData['line'];
                $commentPos = strpos( $line, '#' );
                $origLine = $line;
                // Get rid of any comments before we check the line
                if ( $commentPos !== false )
                {
                    $line = substr( $line, 0, $commentPos );
                }
                $trimLine = trim( $line );
                // Skip empty lines
                if ( strlen( $trimLine ) == 0 )
                    continue;

//                 print( "Line: '$line'\n" );

                $unicodeData = false;

                $sourceValue = false;
                $sourceEndValue = false;
                $destinationValues = false;
                $transposeValue = false;
                $transposeAdd = true;
                $moduloValue = 1;
                // source, marker, range_input, range_marker, map_input, transpose_input, replace_input
                $state = 'source';
                // map, transpose, replace
                $type = false;

                $len = strlen( $line );
                if ( preg_match( '#^(.+):[ \t]*$#', $line, $matches ) )
                {
                    $identifier = $matches[1];
                    if ( !preg_match( '#^[a-zA-Z_-][a-zA-Z0-9_-]*$#', $identifier ) )
                    {
                        $this->warning( "Invalid identifier '$identifier', can only contain a-z, a-Z - and _",
                                      array( 'file' => $filename, 'from' => array( $linePos, strlen( $identifier ) ) ) );
                        $identifier = false;
                        continue;
                    }
//                     print( "identifier '$identifier'\n" );
                    continue;
                }
                else if ( $identifier === false )
                {
                    $this->warning( "No identifier defined yet, skipping: '" . $line . "'",
                                    array( 'file' => $filename, 'from' => array( $linePos, 0 ) ) );
                    continue;
                }
                else
                {
                    $pos = 0;
                    $col = 0;
                    $failed = false;
                    while ( $pos < $len )
                    {
                        while ( $pos < $len and
                                ( $line[$pos] == ' ' or
                                  $line[$pos] == "\t" ) )
                        {
                            ++$pos;
                        }
                        if ( $pos >= $len )
                            break;

                        $char = $line[$pos];
                        $unicodeData = false;
                        if ( $char == '"' )
                        {
                            $delimiterPos = $pos;
                            while ( $delimiterPos < $len )
                            {
                                $delimiterPos = strpos( $line, '"', $delimiterPos + 1 );
                                if ( $delimiterPos === false or
                                     $delimiterPos <= $pos + 1 or
                                     $line[$delimiterPos - 1] != "\\" )
                                    break;
                            }
                            if ( $delimiterPos === false )
                            {
                                $this->warning( "No end-quote found for line, skipping: '$line'",
                                                array( 'file' => $filename,
                                                       'from' => array( $linePos, $pos ),
                                                       'to' => array( $linePos, strlen( $line ) ) ) );
                                $pos = $len;
                                $failed = true;
                                break;
                            }
                            $str = str_replace( array( "\\\"", "\\\\" ),
                                                array( "\"", "\\" ),
                                                substr( $line, $pos + 1, $delimiterPos - $pos - 1 ) );
//                             print( "string '$str'\n" );
                            $pos = $delimiterPos + 1;
                            $unicodeData = array( 'value' => $str,
                                                  'type' => 'string' );
                        }
                        else if ( $char == 'U' and
                             $pos + 1 < $len and
                             $line[$pos + 1] == '+' )
                        {
                            $hexPos = $pos + 2;
                            if ( $hexPos + 4 > $len )
                            {
                                $col = $hexPos;
                                $this->warning( "Found U+ value with " . ( 4 - ( $len - $hexPos ) ) . " missing hex numbers",
                                                array( 'file' => $filename,
                                                       'from' => array( $linePos, $hexPos ) ) );
                                $failed = true;
                                $pos = $hexPos;
                                break;
                            }
                            $hasHexValues = true;
                            for ( $offset = 0; $offset < 4; ++$offset )
                            {
                                $hexChar = $line[$hexPos + $offset];
                                if ( $hexChar == ' ' or
                                     $hexChar == "\t" )
                                {
                                    $col = $hexPos + $offset;
                                    $hasHexValues = false;
                                    $this->warning( "Found U+ value with " . ( 4 - $offset ) . " missing hex numbers",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $hexPos ),
                                                           'to' => array( $linePos, $hexPos + $offset ) ) );
                                    $failed = true;
                                    $pos = $hexPos + $offset;
                                    break;
                                }
                                if ( strpos( $hexValues, $hexChar ) === false )
                                {
                                    $col = $hexPos + $offset;
                                    $hasHexValues = false;
                                    $this->warning( "Found U+ value with invalid hex numbers ($hexChar)",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $hexPos ),
                                                           'to' => array( $linePos, $hexPos + $offset ) ) );
                                    $pos = $hexPos + $offset;
                                    $failed = true;
                                    break;
                                }
                            }
                            if ( $failed )
                                break;
                            if ( $hasHexValues )
                            {
                                $unicodeValue = hexdec( substr( $line, $hexPos, 4 ) );
                                $unicodeData = array( 'value' => $unicodeValue,
                                                      'type' => 'unicode' );
//                                 print( "unicode U+ '$unicodeValue'\n" );
                            }
                            $pos = $hexPos + 4;
                        }
                        else if ( strpos( $hexValues, $char ) !== false and
                                  $pos + 1 < $len and
                                  strpos( $hexValues, $line[$pos + 1] ) !== false )
                        {
                            $hexPos = $pos;
                            if ( $hexPos + 2 > $len )
                            {
                                $col = $len;
                                $this->warning( "Found ASCII value with " . ( 2 - ( $len - $hexPos ) ) . " missing hex numbers",
                                                array( 'file' => $filename,
                                                       'from' => array( $linePos, $hexPos ) ) );
                                $pos = $hexPos;
                                $failed = true;
                                break;
                            }
                            $hasHexValues = true;
                            for ( $offset = 0; $offset < 2; ++$offset )
                            {
                                $hexChar = $line[$hexPos + $offset];
                                if ( $hexChar == ' ' or
                                     $hexChar == "\t" )
                                {
                                    $col = $hexPos + $offset;
                                    $hasHexValues = false;
                                    $this->warning( "Found ASCII value with " . ( 2 - $offset ) . " missing hex numbers",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $hexPos ),
                                                           'to' => array( $linePos, $hexPos + $offset ) ) );
                                    $pos = $hexPos + $offset;
                                    $failed = true;
                                    break;
                                }
                                if ( strpos( $hexValues, $hexChar ) === false )
                                {
                                    $col = $hexPos + $offset;
                                    $hasHexValues = false;
                                    $this->warning( "Found ASCII value with invalid hex numbers ($hexChar)",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $hexPos ),
                                                           'to' => array( $linePos, $hexPos + $offset ) ) );
                                    $pos = $hexPos + $offset;
                                    $failed = true;
                                    break;
                                }
                            }
                            if ( $failed )
                                break;
                            if ( $hasHexValues )
                            {
                                $asciiValue = hexdec( substr( $line, $hexPos, 4 ) );
//                                 print( "unicode ASCII '$asciiValue'\n" );
                                $unicodeData = array( 'value' => $asciiValue,
                                                      'type' => 'ascii' );
                            }
                            $pos = $hexPos + 2;
                        }
                        else if ( substr( $line, $pos, 6 ) == 'remove' )
                        {
//                             print( "remove character\n" );
                            $unicodeData = array( 'value' => false,
                                                  'type' => 'remove' );
                            $pos += 6;
                        }
                        else if ( substr( $line, $pos, 4 ) == 'keep' )
                        {
//                             print( "keep character\n" );
                            $unicodeData = array( 'value' => true,
                                                  'type' => 'keep' );
                            $pos += 4;
                        }

                        if ( $unicodeData )
                        {
//                             print( "data state: $state\n" );
                            // source, marker, range_input, range_marker, map_input, transpose_input, replace_input, transpose_modulo
                            if ( $state == 'source' )
                            {
                                if ( $unicodeData['type'] == 'string' and
                                     strlen( $unicodeData['value'] ) > 1 )
                                {
                                    $this->warning( "Text string with more than one character cannot be used as input value '" . $unicodeData['value'] . "'",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                $sourceValue = $this->extractUnicodeValue( $unicodeData );
                                $state = 'marker';
                            }
                            else if ( $state == 'marker' )
                            {
                                $this->warning( "Source value not expected, a source value has already been extracted at $line" . "[$pos]",
                                                array( 'file' => $filename,
                                                       'from' => array( $linePos, $pos ) ) );
                                $failed = true;
                                break;
                            }
                            else if ( $state == 'range_input' )
                            {
                                if ( $unicodeData['type'] == 'string' and
                                     strlen( $unicodeData['value'] ) > 1 )
                                {
                                    $this->warning( "Text string with more than one character cannot be used as range end value '" . $unicodeData['value'] . "'",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                $sourceEndValue = $this->extractUnicodeValue( $unicodeData );
                                $state = 'range_marker_or_modulo';
                            }
                            else if ( $state == 'range_marker_or_modulo' or
                                      $state == 'range_marker' )
                            {
                                $this->warning( "Range value not expected, a range value has already been extracted at $line" . "[$pos]",
                                                array( 'file' => $filename,
                                                       'from' => array( $linePos, $pos ) ) );
                                $failed = true;
                                break;
                            }
                            else if ( $state == 'map_input' )
                            {
                                if ( !is_array( $destinationValues ) )
                                    $destinationValues = array();
                                $destinationValues = array_merge( $destinationValues,
                                                                  $this->extractUnicodeValues( $unicodeData ) );
                                $type = 'map';
                            }
                            else if ( $state == 'replace_input' )
                            {
                                if ( !is_array( $destinationValues ) )
                                    $destinationValues = array();
                                $destinationValues = array_merge( $destinationValues,
                                                                  $this->extractUnicodeValues( $unicodeData ) );
                                $type = 'replace';
                            }
                            else if ( $state == 'transpose_input' )
                            {
                                if ( $unicodeData['type'] == 'string' and
                                     strlen( $unicodeData['value'] ) > 1 )
                                {
                                    $this->warning( "Text string with more than one character cannot be used as transpose value '" . $unicodeData['value'] . "'",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                $transposeValue = $this->extractUnicodeValue( $unicodeData );
                                $type = 'transpose';
                            }
                            else if ( $state == 'transpose_modulo' )
                            {
                                if ( $unicodeData['type'] == 'string' and
                                     strlen( $unicodeData['value'] ) > 1 )
                                {
                                    $this->warning( "Text string with more than one character cannot be used as transpose modulo value '" . $unicodeData['value'] . "'",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                $moduloValue = $this->extractUnicodeValue( $unicodeData );
                                if ( $moduloValue == 0 )
                                {
                                    $this->error( "Modulo value of 0 is not allowed, 1 will be used instead",
                                                  array( 'file' => $filename,
                                                         'from' => array( $linePos, $pos ) ) );
                                    // Note: There is another 0 check in generateSimpleMappingTable()
                                }
//                                 print( "modulo value=$moduloValue\n" );
                                $state = 'range_marker';
                            }
                        }
                        else if ( !$failed )
                        {
//                             print( "command state: $state\n" );
                            // source, marker, range_input, range_marker, map_input, transpose_input, replace_input
                            if ( $state == 'source' )
                            {
                                if ( $char == '=' )
                                {
                                    $this->warning( "Cannot use map marker $char without prior character value",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else if ( $char == '+' or
                                          $char == '-' )
                                {
                                    $this->warning( "Cannot use range marker $char without prior character value",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else
                                {
                                    $this->warning( "Unknown character '$char', expecting input value",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                            }
                            else if ( $state == 'marker' )
                            {
                                if ( $char == '=' )
                                {
                                    $state = 'map_input';
                                    ++$pos;
                                }
                                else if ( $char == '-' )
                                {
                                    $state = 'range_input';
                                    ++$pos;
                                }
                                else if ( $char == '+' )
                                {
                                    $this->warning( "Cannot use range marker $char without prior character value",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else
                                {
                                    $this->warning( "Unknown character '$char', expecting marker",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                            }
                            else if ( $state == 'range_marker_or_modulo' or
                                      $state == 'range_marker' )
                            {
                                if ( $state == 'range_marker_or_modulo' and
                                     $char == '%' )
                                {
//                                     print( "found modulo marker\n" );
                                    // Look for modulo value
                                    $state = 'transpose_modulo';
                                    ++$pos;
                                }
                                else if ( $char == '=' )
                                {
                                    $state = 'replace_input';
                                    ++$pos;
                                }
                                else if ( $char == '-' or
                                          $char == '+' )
                                {
                                    $transposeAdd = ( $char == '+' ? true : false );
                                    $state = 'transpose_input';
                                    ++$pos;
                                }
                                else
                                {
                                    $this->warning( "Unknown character '$char', expecting range end value",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                            }
                            else if ( $state == 'map_input' )
                            {
                                if ( $char == '=' )
                                {
                                    $this->warning( "Duplicate mapping marker $char",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else if ( $char == '-' or
                                          $char == '+' )
                                {
                                    $this->warning( "Already mapping values, cannot use range/transpose marker $char",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else
                                {
                                    $this->warning( "Unknown character '$char', expecting output values",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                            }
                            else if ( $state == 'transpose_modulo' )
                            {
                                if ( $char == '%' )
                                {
                                    $this->warning( "Modulo marker already used, cannot use $char",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else if ( $char == '-' or
                                          $char == '+' )
                                {
                                    $this->warning( "Transpose marker $char used, but no modulo value has been found yet",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else
                                {
                                    $this->warning( "Unknown character '$char', expecting modulo value",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                            }
                            else if ( $state == 'transpose_input' )
                            {
                                if ( $char == '=' )
                                {
                                    $this->warning( "Already transposing, cannot use mapping marker $char",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else if ( $char == '-' or
                                          $char == '+' )
                                {
                                    $this->warning( "Duplicate transpose marker $char",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else
                                {
                                    $this->warning( "Unknown character '$char', expecting transpose value",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                            }
                            else if ( $state == 'replace_input' )
                            {
                                if ( $char == '=' )
                                {
                                    $this->warning( "Already replacing, cannot use mapping marker $char",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else if ( $char == '-' or
                                          $char == '+' )
                                {
                                    $this->warning( "Already replacing, cannot use transpose marker $char",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                                else
                                {
                                    $this->warning( "Unknown character '$char', expecting replace value",
                                                    array( 'file' => $filename,
                                                           'from' => array( $linePos, $pos ) ) );
                                    $failed = true;
                                    break;
                                }
                            }
                        }
                    }
                    if ( !$failed )
                    {
                        if ( $identifier )
                        {
//                             print( "\nGot type '$type'\n" );
//                            if ( is_array( $destinationValues ) )
//                                $destinationValues = array_diff( $destinationValues, array( '' ) );

                            if ( !isset( $tbl[$identifier] ) )
                                $tbl[$identifier] = array();

                            if ( $type == 'map' )
                            {
//                                 print( "***mapping***:\n" . $sourceValue . ' => ' . implode( ', ', $destinationValues ) . "\n\n" );
                                $this->appendDirectMapping( $tbl[$identifier], $identifier, $sourceValue, $destinationValues );
                            }
                            else if ( $type == 'replace' )
                            {
//                                 print( "***replacing***:\n" . $sourceValue . ' - ' . $sourceEndValue . ' => ' . implode( ', ', $destinationValues ) . "\n\n" );
                                $this->appendReplaceMapping( $tbl[$identifier], $identifier, $sourceValue, $sourceEndValue, $destinationValues );
                            }
                            else if ( $type == 'transpose' )
                            {
//                                 print( "***transposing***:\n" . $sourceValue . ' - ' . $sourceEndValue . ' % ' . $moduloValue . ' + ' . $transposeValue . "\n\n" );
                                $this->appendTransposeMapping( $tbl[$identifier], $identifier, $sourceValue, $sourceEndValue, $transposeValue, $transposeAdd, $moduloValue );
                            }
                        }
//                         else
//                         {
//                             print( "No identifier found yet, skipping entry!!!!!!!!!!\n" );
//                         }
                    }
                    else
                    {
//                         $this->warning( "Failed adding mapper",
//                                         array( 'file' => $filename,
//                                                'from' => array( $linePos, $pos ) ) );
                    }
                }
            }

            // Here we read more data from the file, appending to
            // the $buffer variable
            if ( !feof( $fd ) )
            {
                $buffer .= fread( $fd, 4096 );

                // Make sure we have Unix endline characters
                $buffer = preg_replace( "#(\r\n|\r|\n)#", "\n", $buffer );
            }
            ++$i;
        }

        fclose( $fd );

        $this->TransformationTables = array_merge( $this->TransformationTables, $tbl );
    }
592291
public function setCharset(ConnectionInterface $con, $charset)
    {
        switch (strtolower($charset)) {
            case 'utf-8':
                $con->setAttribute(\PDO::SQLSRV_ATTR_ENCODING, \PDO::SQLSRV_ENCODING_UTF8);
                break;
            case 'system':
                $con->setAttribute(\PDO::SQLSRV_ATTR_ENCODING, \PDO::SQLSRV_ENCODING_SYSTEM);
                break;
            default:
                throw new UnsupportedEncodingException('only utf-8 or system encoding are supported by the pdo_sqlsrv driver');
        }
    }
1208889
public static function convert_from($data, $charset = 'cp1251')
	{
		if (! ReflectionTypeHint::isValid()) return false;
		return self::_convert($data, $charset, 'UTF-8');
	}
966248
function utf8_encode($str, $charset) {

		if ($charset === 'utf-8') {
			return $str;
		}

			// Charset is case-insensitive.
		if ($this->initCharset($charset)) { // Parse conv. table if not already...
			$strLen = strlen($str);
			$outStr = '';

			for ($a = 0; $a < $strLen; $a++) { // Traverse each char in string.
				$chr = substr($str, $a, 1);
				$ord = ord($chr);
				if (isset($this->twoByteSets[$charset])) { // If the charset has two bytes per char
					$ord2 = ord($str{$a + 1});
					$ord = $ord << 8 | $ord2; // assume big endian

					if (isset($this->parsedCharsets[$charset]['local'][$ord])) { // If the local char-number was found in parsed conv. table then we use that, otherwise 127 (no char?)
						$outStr .= $this->parsedCharsets[$charset]['local'][$ord];
					} else {
						$outStr .= chr($this->noCharByteVal);
					} // No char exists
					$a++;
				} elseif ($ord > 127) { // If char has value over 127 it's a multibyte char in UTF-8
					if (isset($this->eucBasedSets[$charset])) { // EUC uses two-bytes above 127; we get both and advance pointer and make $ord a 16bit int.
						if ($charset != 'shift_jis' || ($ord < 0xA0 || $ord > 0xDF)) { // Shift-JIS: chars between 160 and 223 are single byte
							$a++;
							$ord2 = ord(substr($str, $a, 1));
							$ord = $ord * 256 + $ord2;
						}
					}

					if (isset($this->parsedCharsets[$charset]['local'][$ord])) { // If the local char-number was found in parsed conv. table then we use that, otherwise 127 (no char?)
						$outStr .= $this->parsedCharsets[$charset]['local'][$ord];
					} else {
						$outStr .= chr($this->noCharByteVal);
					} // No char exists
				} else {
					$outStr .= $chr;
				} // ... otherwise it's just ASCII 0-127 and one byte. Transparent
			}
			return $outStr;
		}
	}
238189
public function isEncoded( $fileName )
	{
		if( file_exists( $fileName ) )
		{
			$fp	= fopen( $fileName, "r" );
			$code	= fgets( $fp, strlen( $this->outcodePrefix ) );
			if( $code == $this->outcodePrefix )
				return TRUE;
		}
		return FALSE;
	}
595057
public static function changeEncode($data, $in_charset = 'GBK', $out_charset = 'UTF-8')
    {
        if (\is_array($data)) {

            foreach ($data as $key => $value) {
                $data[$key] = self::changeEncode($value, $in_charset, $out_charset);
            }

            return $data;
        }

        if (\function_exists('mb_convert_encoding')) {
            return mb_convert_encoding($data, $out_charset, $in_charset);
        }

        return iconv($in_charset, $out_charset . '/' . '/IGNORE', $data);
    }