Clean UTF-8 strings

Ensures strings contain only valid UTF-8 characters.

@param    string $str String to clean

@return    string
186785
public function clean_string($str)
    {
        if ($this->is_ascii($str) === FALSE) {
            if (MB_ENABLED) {
                $str = mb_convert_encoding($str, 'UTF-8', 'UTF-8');
            } elseif (ICONV_ENABLED) {
                $str = @iconv('UTF-8', 'UTF-8//IGNORE', $str);
            }
        }

        return $str;
    }
82265
private function _strtoupper($str)
    {
        $str = html_entity_decode($str, ENT_COMPAT);

        if (function_exists('mb_strtoupper')) {
            $str = mb_strtoupper($str, 'UTF-8');
        } else {
            $str = strtoupper($str);
        }

        $str = htmlspecialchars($str, ENT_COMPAT);

        return $str;
    }
1354471
private function _repack_utf7(string $str): string
    {
        return (string) \preg_replace_callback(
            '#\+([\\p{L}0-9]+)\-#ui',
            function ($matches) {
                return $this->_repack_utf7_callback($matches);
            },
            $str
        );
    }
627632
protected function clean($string)
    {
        if ($this->format == 'xml') {
            return $string;
        }

        $string = html_entity_decode($string, ENT_QUOTES, 'UTF-8');
        $string = htmlentities($string, ENT_QUOTES, 'UTF-8');

        if (function_exists('mb_convert_encoding')) {
            $string = mb_convert_encoding($string, 'UTF-8');
        } else {
            $string = utf8_decode($string);
        }

        return $string;
    }
1062683
private function charPad($str)
    {
        $l = strlen($str)*8;     // # of bits from input string
        $str .= "\x80";          // append the "1" bit followed by 7 0's
        $k = (512 - (($l + 8 + 64) % 512)) / 8;   // # of 0 bytes to append
        $k += 4;    // PHP String's will never exceed (2^31)-1, so 1st 32bits of
                    // the 64-bit value representing $l can be all 0's
        for ($x = 0; $x < $k; $x++) {
            $str .= "\0";
        }
        // append the last 32-bits representing the # of bits from input string ($l)
        $str .= chr((($l >> 24) & 0xFF));
        $str .= chr((($l >> 16) & 0xFF));
        $str .= chr((($l >> 8) & 0xFF));
        $str .= chr(($l & 0xFF));
        return $str;
    }
1008919
public static function safe($str)
    {
        if(is_array($str)){
            foreach($str as $key=>$val){
                $str[$key] = self::safe($val);
            }
            return $str;
        }
        return htmlspecialchars($str, ENT_QUOTES);
    }
626141
public static function normalize($str)
    {
        $str = strtr($str, ['ő' => 'o', 'ű' => 'u', 'Ő' => 'O', 'Ű' => 'U']);
        $a = 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøùúûűüýýþÿŔŕ';
        $b = 'AAAAAAACEEEEIIIIDNOOOOOOOUUUUUYbsaaaaaaaceeeeiiiidnooooooouuuuuyybyRr';
        $str = utf8_decode($str);
        $str = str_replace('?', '', $str);
        $str = strtr($str, utf8_decode($a), $b);
        return utf8_encode($str);
    }
1245860
public static function toAscii(string $str) : string
    {
        if (preg_match("/[\x80-\xFF]/", $str)) {
            // Grab the transliteration table since we'll need it.
            if (null === static::$ascii) {
                static::$ascii = unserialize(file_get_contents(__DIR__ . '/str/resources/transliteration_table.ser'));
            }

            $str = \Normalizer::normalize($str, \Normalizer::NFKD);
            $str = preg_replace('/\p{Mn}+/u', '', $str);
            $str = str_replace(static::$ascii[0], static::$ascii[1], $str);
            $str = iconv('UTF-8', 'ASCII' . ('glibc' !== ICONV_IMPL ? '//IGNORE' : '') . '//TRANSLIT', $str);
        }

        return $str;
    }
807350
public static function Utf7ImapToUtf8($str)
    {
        if ($str instanceof Horde_Imap_Client_Mailbox) {
            return $str->utf8;
        }

        $str = strval($str);

        /* Try mbstring, if available, which should be faster. Don't use the
         * IMAP utf7_* functions because they are known to be buggy. */
        if (is_null(self::$_mbstring)) {
            self::$_mbstring = extension_loaded('mbstring');
        }
        if (self::$_mbstring) {
            return @mb_convert_encoding($str, 'UTF-8', 'UTF7-IMAP');
        }

        $p = '';
        $ptr = &self::$_index64;

        for ($i = 0, $u7len = strlen($str); $u7len > 0; ++$i, --$u7len) {
            $u7 = $str[$i];
            if ($u7 === '&') {
                $u7 = $str[++$i];
                if (--$u7len && ($u7 === '-')) {
                    $p .= '&';
                    continue;
                }

                $ch = 0;
                $k = 10;
                for (; $u7len > 0; ++$i, --$u7len) {
                    $u7 = $str[$i];

                    if ((ord($u7) & 0x80) || ($b = $ptr[ord($u7)]) === -1) {
                        break;
                    }

                    if ($k > 0) {
                        $ch |= $b << $k;
                        $k -= 6;
                    } else {
                        $ch |= $b >> (-$k);
                        if ($ch < 0x80) {
                            /* Printable US-ASCII */
                            if ((0x20 <= $ch) && ($ch < 0x7f)) {
                                throw new Horde_Imap_Client_Exception(
                                    Horde_Imap_Client_Translation::r("Error converting UTF7-IMAP string."),
                                    Horde_Imap_Client_Exception::UTF7IMAP_CONVERSION
                                );
                            }
                            $p .= chr($ch);
                        } else if ($ch < 0x800) {
                            $p .= chr(0xc0 | ($ch >> 6)) .
                                  chr(0x80 | ($ch & 0x3f));
                        } else {
                            $p .= chr(0xe0 | ($ch >> 12)) .
                                  chr(0x80 | (($ch >> 6) & 0x3f)) .
                                  chr(0x80 | ($ch & 0x3f));
                        }

                        $ch = ($b << (16 + $k)) & 0xffff;
                        $k += 10;
                    }
                }

                /* Non-zero or too many extra bits -OR-
                 * Base64 not properly terminated -OR-
                 * Adjacent Base64 sections. */
                if (($ch || ($k < 6)) ||
                    (!$u7len || $u7 !== '-') ||
                    (($u7len > 2) &&
                     ($str[$i + 1] === '&') &&
                     ($str[$i + 2] !== '-'))) {
                    throw new Horde_Imap_Client_Exception(
                        Horde_Imap_Client_Translation::r("Error converting UTF7-IMAP string."),
                        Horde_Imap_Client_Exception::UTF7IMAP_CONVERSION
                    );
                }
            } elseif ((ord($u7) < 0x20) || (ord($u7) >= 0x7f)) {
                /* Not printable US-ASCII */
                throw new Horde_Imap_Client_Exception(
                    Horde_Imap_Client_Translation::r("Error converting UTF7-IMAP string."),
                    Horde_Imap_Client_Exception::UTF7IMAP_CONVERSION
                );
            } else {
                $p .= $u7;
            }
        }

        return $p;
    }
1348183
public function decode($str) {

		$str = strtoupper($str);

		$l = strlen($str);
		$n = 0;
		$b = 0;

		$decoded = null;

		for ($i = 0; $i < $l; $i++) {
			$n = $n << 5;
			$n = $n + $this->decodeTable[$str[$i]];
			$b = $b + 5;

			if ($b >= 8) {
				$b = $b - 8;
				$decoded .= chr(($n & (0xFF << $b)) >> $b);
			}
		}

		return $decoded;
	}
1139605
function UTF8toSubset($str)
	{
		$ret = '<';
		// $str = preg_replace('/'.preg_quote($this->aliasNbPg,'/').'/', chr(7), $str );	// mPDF 6 deleted
		// $str = preg_replace('/'.preg_quote($this->aliasNbPgGp,'/').'/', chr(8), $str );	// mPDF 6 deleted
		$unicode = $this->UTF8StringToArray($str);
		$orig_fid = $this->CurrentFont['subsetfontids'][0];
		$last_fid = $this->CurrentFont['subsetfontids'][0];
		foreach ($unicode as $c) {
			/* 	// mPDF 6 deleted
			  if ($c == 7 || $c == 8) {
			  if ($orig_fid != $last_fid) {
			  $ret .= '> Tj /F'.$orig_fid.' '.$this->FontSizePt.' Tf <';
			  $last_fid = $orig_fid;
			  }
			  if ($c == 7) { $ret .= $this->aliasNbPgHex; }
			  else { $ret .= $this->aliasNbPgGpHex; }
			  continue;
			  }
			 */
			if (!$this->_charDefined($this->CurrentFont['cw'], $c)) {
				$c = 0;
			} // mPDF 6
			for ($i = 0; $i < 99; $i++) {
				// return c as decimal char
				$init = array_search($c, $this->CurrentFont['subsets'][$i]);
				if ($init !== false) {
					if ($this->CurrentFont['subsetfontids'][$i] != $last_fid) {
						$ret .= '> Tj /F' . $this->CurrentFont['subsetfontids'][$i] . ' ' . $this->FontSizePt . ' Tf <';
						$last_fid = $this->CurrentFont['subsetfontids'][$i];
					}
					$ret .= sprintf("%02s", strtoupper(dechex($init)));
					break;
				} // TrueType embedded SUBSETS
				elseif (count($this->CurrentFont['subsets'][$i]) < 255) {
					$n = count($this->CurrentFont['subsets'][$i]);
					$this->CurrentFont['subsets'][$i][$n] = $c;
					if ($this->CurrentFont['subsetfontids'][$i] != $last_fid) {
						$ret .= '> Tj /F' . $this->CurrentFont['subsetfontids'][$i] . ' ' . $this->FontSizePt . ' Tf <';
						$last_fid = $this->CurrentFont['subsetfontids'][$i];
					}
					$ret .= sprintf("%02s", strtoupper(dechex($n)));
					break;
				} elseif (!isset($this->CurrentFont['subsets'][($i + 1)])) {
					// TrueType embedded SUBSETS
					$this->CurrentFont['subsets'][($i + 1)] = [0 => 0];
					$new_fid = count($this->fonts) + $this->extraFontSubsets + 1;
					$this->CurrentFont['subsetfontids'][($i + 1)] = $new_fid;
					$this->extraFontSubsets++;
				}
			}
		}
		$ret .= '>';
		if ($last_fid != $orig_fid) {
			$ret .= ' Tj /F' . $orig_fid . ' ' . $this->FontSizePt . ' Tf <> ';
		}
		return $ret;
	}
487684
public function decode($str)
	{
		if ($this->decodeHtmlEntities && strpos($str, '&') !== false)
		{
			$str = html_entity_decode($str, ENT_QUOTES, 'UTF-8');
		}
		$str = str_replace("\x1A", '', $str);

		if ($this->hasEscapedChars)
		{
			$str = strtr(
				$str,
				[
					"\x1B0" => '!', "\x1B1" => '"', "\x1B2" => "'", "\x1B3" => '(',
					"\x1B4" => ')', "\x1B5" => '*', "\x1B6" => '[', "\x1B7" => '\\',
					"\x1B8" => ']', "\x1B9" => '^', "\x1BA" => '_', "\x1BB" => '`',
					"\x1BC" => '~'
				]
			);
		}

		return $str;
	}
112922
public static function convertToUTF8($str)
    {
        $encoding = mb_detect_encoding($str, ['UTF-8', 'ISO-8859-1', 'WINDOWS-1252'], true);

        switch ($encoding) {
            case 'ISO-8859-1':
                $newStr = utf8_encode($str);
                break;
            /* Unreachable code. Not sure yet how we can improve this
             * situation.
            case 'WINDOWS-1252' :
                $newStr = iconv('cp1252', 'UTF-8', $str);
                break;
             */
            default:
                 $newStr = $str;
        }

        // Removing any control characters
        return preg_replace('%(?:[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F])%', '', $newStr);
    }
130806
private function cleanInputData($str) {
        if (is_array($str)) {
            $new_array = [];
            foreach ($str as $key => $val) {
                $new_array [ $this->cleanInputKeys($key) ] = $this->cleanInputData($val);
            }

            return $new_array;
        }
        $str = self::$xss_cleaner->xss_clean($str);
        // Standardize newlines
        if (strpos($str, "\r") !== false) {
            $str = str_replace(["\r\n", "\r"], "\n", $str);
        }

        return $str;
    }
115232
final public static function safeEncodeStr(string $str): string
    {
        return preg_replace_callback('/&#([a-z\d]+);/i', function ($m) {
            $value = (string)$m[0];
            $value = mb_convert_encoding($value, 'UTF-8', 'HTML-ENTITIES');
            return $value;
        }, $str);
    }
682293
protected function filterString(string $str): string
    {
        $invalid_chars = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x7F";
        if (strlen($str) !== strcspn($str, $invalid_chars)) {
            throw new Exception(sprintf('the submitted string `%s` contains invalid characters', $str));
        }

        return $str;
    }
535643
private function _clean_input_data($str) {
		if (is_array($str)) {
			$new_array = array();
			foreach ($str as $key => $val) {
				$new_array[$this->_clean_input_keys($key)] = $this->_clean_input_data($val);
			}
			return $new_array;
		}
		
		/*
		 * We strip slashes if magic quotes is on to keep things consistent
		 * NOTE: In PHP 5.4 get_magic_quotes_gpc() will always return 0 and it
		 * will probably not exist in future versions at all.
		 */
		if (get_magic_quotes_gpc()) {
			$str = stripslashes($str);
		}
		
		// Remove control characters
		$str = Security::remove_invisible_characters($str);
		
		// Should we filter the input data?
		if ($this->_enable_xss === TRUE) {
			$str = Security::xss_clean($str);
		}
		
		// Standardize newlines if needed
		if ($this->_standardize_newlines == TRUE) {
			if (strpos($str, "\r") !== FALSE) {
				$str = str_replace(array("\r\n", "\r", "\r\n\n"), PHP_EOL, $str);
			}
		}
		
		return $str;
	}