PageRenderTime 44ms CodeModel.GetById 12ms RepoModel.GetById 0ms app.codeStats 0ms

/program/include/rcube_charset.php

https://github.com/netconstructor/roundcubemail
PHP | 791 lines | 546 code | 93 blank | 152 comment | 227 complexity | 59959cd52430643431e6475c9b266238 MD5 | raw file
Possible License(s): GPL-3.0, LGPL-2.1
  1. <?php
  2. /*
  3. +-----------------------------------------------------------------------+
  4. | program/include/rcube_charset.php |
  5. | |
  6. | This file is part of the Roundcube Webmail client |
  7. | Copyright (C) 2005-2012, The Roundcube Dev Team |
  8. | Copyright (C) 2011-2012, Kolab Systems AG |
  9. | Copyright (C) 2000 Edmund Grimley Evans <edmundo@rano.org> |
  10. | |
  11. | Licensed under the GNU General Public License version 3 or |
  12. | any later version with exceptions for skins & plugins. |
  13. | See the README file for a full license statement. |
  14. | |
  15. | PURPOSE: |
  16. | Provide charset conversion functionality |
  17. | |
  18. +-----------------------------------------------------------------------+
  19. | Author: Thomas Bruederli <roundcube@gmail.com> |
  20. | Author: Aleksander Machniak <alec@alec.pl> |
  21. +-----------------------------------------------------------------------+
  22. */
  23. /**
  24. * Character sets conversion functionality
  25. *
  26. * @package Framework
  27. * @subpackage Core
  28. * @author Thomas Bruederli <roundcube@gmail.com>
  29. * @author Aleksander Machniak <alec@alec.pl>
  30. * @author Edmund Grimley Evans <edmundo@rano.org>
  31. */
  32. class rcube_charset
  33. {
  34. // Aliases: some of them from HTML5 spec.
  35. static public $aliases = array(
  36. 'USASCII' => 'WINDOWS-1252',
  37. 'ANSIX31101983' => 'WINDOWS-1252',
  38. 'ANSIX341968' => 'WINDOWS-1252',
  39. 'UNKNOWN8BIT' => 'ISO-8859-15',
  40. 'UNKNOWN' => 'ISO-8859-15',
  41. 'USERDEFINED' => 'ISO-8859-15',
  42. 'KSC56011987' => 'EUC-KR',
  43. 'GB2312' => 'GBK',
  44. 'GB231280' => 'GBK',
  45. 'UNICODE' => 'UTF-8',
  46. 'UTF7IMAP' => 'UTF7-IMAP',
  47. 'TIS620' => 'WINDOWS-874',
  48. 'ISO88599' => 'WINDOWS-1254',
  49. 'ISO885911' => 'WINDOWS-874',
  50. 'MACROMAN' => 'MACINTOSH',
  51. '77' => 'MAC',
  52. '128' => 'SHIFT-JIS',
  53. '129' => 'CP949',
  54. '130' => 'CP1361',
  55. '134' => 'GBK',
  56. '136' => 'BIG5',
  57. '161' => 'WINDOWS-1253',
  58. '162' => 'WINDOWS-1254',
  59. '163' => 'WINDOWS-1258',
  60. '177' => 'WINDOWS-1255',
  61. '178' => 'WINDOWS-1256',
  62. '186' => 'WINDOWS-1257',
  63. '204' => 'WINDOWS-1251',
  64. '222' => 'WINDOWS-874',
  65. '238' => 'WINDOWS-1250',
  66. 'MS950' => 'CP950',
  67. 'WINDOWS949' => 'UHC',
  68. );
  69. /**
  70. * Catch an error and throw an exception.
  71. *
  72. * @param int Level of the error
  73. * @param string Error message
  74. */
  75. public static function error_handler($errno, $errstr)
  76. {
  77. throw new ErrorException($errstr, 0, $errno);
  78. }
  79. /**
  80. * Parse and validate charset name string (see #1485758).
  81. * Sometimes charset string is malformed, there are also charset aliases
  82. * but we need strict names for charset conversion (specially utf8 class)
  83. *
  84. * @param string $input Input charset name
  85. *
  86. * @return string The validated charset name
  87. */
  88. public static function parse_charset($input)
  89. {
  90. static $charsets = array();
  91. $charset = strtoupper($input);
  92. if (isset($charsets[$input])) {
  93. return $charsets[$input];
  94. }
  95. $charset = preg_replace(array(
  96. '/^[^0-9A-Z]+/', // e.g. _ISO-8859-JP$SIO
  97. '/\$.*$/', // e.g. _ISO-8859-JP$SIO
  98. '/UNICODE-1-1-*/', // RFC1641/1642
  99. '/^X-/', // X- prefix (e.g. X-ROMAN8 => ROMAN8)
  100. ), '', $charset);
  101. if ($charset == 'BINARY') {
  102. return $charsets[$input] = null;
  103. }
  104. // allow A-Z and 0-9 only
  105. $str = preg_replace('/[^A-Z0-9]/', '', $charset);
  106. if (isset(self::$aliases[$str])) {
  107. $result = self::$aliases[$str];
  108. }
  109. // UTF
  110. else if (preg_match('/U[A-Z][A-Z](7|8|16|32)(BE|LE)*/', $str, $m)) {
  111. $result = 'UTF-' . $m[1] . $m[2];
  112. }
  113. // ISO-8859
  114. else if (preg_match('/ISO8859([0-9]{0,2})/', $str, $m)) {
  115. $iso = 'ISO-8859-' . ($m[1] ? $m[1] : 1);
  116. // some clients sends windows-1252 text as latin1,
  117. // it is safe to use windows-1252 for all latin1
  118. $result = $iso == 'ISO-8859-1' ? 'WINDOWS-1252' : $iso;
  119. }
  120. // handle broken charset names e.g. WINDOWS-1250HTTP-EQUIVCONTENT-TYPE
  121. else if (preg_match('/(WIN|WINDOWS)([0-9]+)/', $str, $m)) {
  122. $result = 'WINDOWS-' . $m[2];
  123. }
  124. // LATIN
  125. else if (preg_match('/LATIN(.*)/', $str, $m)) {
  126. $aliases = array('2' => 2, '3' => 3, '4' => 4, '5' => 9, '6' => 10,
  127. '7' => 13, '8' => 14, '9' => 15, '10' => 16,
  128. 'ARABIC' => 6, 'CYRILLIC' => 5, 'GREEK' => 7, 'GREEK1' => 7, 'HEBREW' => 8
  129. );
  130. // some clients sends windows-1252 text as latin1,
  131. // it is safe to use windows-1252 for all latin1
  132. if ($m[1] == 1) {
  133. $result = 'WINDOWS-1252';
  134. }
  135. // if iconv is not supported we need ISO labels, it's also safe for iconv
  136. else if (!empty($aliases[$m[1]])) {
  137. $result = 'ISO-8859-'.$aliases[$m[1]];
  138. }
  139. // iconv requires convertion of e.g. LATIN-1 to LATIN1
  140. else {
  141. $result = $str;
  142. }
  143. }
  144. else {
  145. $result = $charset;
  146. }
  147. $charsets[$input] = $result;
  148. return $result;
  149. }
  150. /**
  151. * Convert a string from one charset to another.
  152. * Uses mbstring and iconv functions if possible
  153. *
  154. * @param string Input string
  155. * @param string Suspected charset of the input string
  156. * @param string Target charset to convert to; defaults to RCMAIL_CHARSET
  157. *
  158. * @return string Converted string
  159. */
  160. public static function convert($str, $from, $to = null)
  161. {
  162. static $iconv_options = null;
  163. static $mbstring_list = null;
  164. static $mbstring_sch = null;
  165. static $conv = null;
  166. $to = empty($to) ? RCMAIL_CHARSET : $to;
  167. $from = self::parse_charset($from);
  168. // It is a common case when UTF-16 charset is used with US-ASCII content (#1488654)
  169. // In that case we can just skip the conversion (use UTF-8)
  170. if ($from == 'UTF-16' && !preg_match('/[^\x00-\x7F]/', $str)) {
  171. $from = 'UTF-8';
  172. }
  173. if ($from == $to || empty($str) || empty($from)) {
  174. return $str;
  175. }
  176. if ($iconv_options === null) {
  177. if (function_exists('iconv')) {
  178. // ignore characters not available in output charset
  179. $iconv_options = '//IGNORE';
  180. if (iconv('', $iconv_options, '') === false) {
  181. // iconv implementation does not support options
  182. $iconv_options = '';
  183. }
  184. }
  185. }
  186. // convert charset using iconv module
  187. if ($iconv_options !== null && $from != 'UTF7-IMAP' && $to != 'UTF7-IMAP') {
  188. // throw an exception if iconv reports an illegal character in input
  189. // it means that input string has been truncated
  190. set_error_handler(array('rcube_charset', 'error_handler'), E_NOTICE);
  191. try {
  192. $_iconv = iconv($from, $to . $iconv_options, $str);
  193. } catch (ErrorException $e) {
  194. $_iconv = false;
  195. }
  196. restore_error_handler();
  197. if ($_iconv !== false) {
  198. return $_iconv;
  199. }
  200. }
  201. if ($mbstring_list === null) {
  202. if (extension_loaded('mbstring')) {
  203. $mbstring_sch = mb_substitute_character();
  204. $mbstring_list = mb_list_encodings();
  205. $mbstring_list = array_map('strtoupper', $mbstring_list);
  206. }
  207. }
  208. // convert charset using mbstring module
  209. if ($mbstring_list !== null) {
  210. $aliases['WINDOWS-1257'] = 'ISO-8859-13';
  211. // it happens that mbstring supports ASCII but not US-ASCII
  212. if (($from == 'US-ASCII' || $to == 'US-ASCII') && !in_array('US-ASCII', $mbstring_list)) {
  213. $aliases['US-ASCII'] = 'ASCII';
  214. }
  215. $mb_from = $aliases[$from] ? $aliases[$from] : $from;
  216. $mb_to = $aliases[$to] ? $aliases[$to] : $to;
  217. // return if encoding found, string matches encoding and convert succeeded
  218. if (in_array($mb_from, $mbstring_list) && in_array($mb_to, $mbstring_list)) {
  219. if (mb_check_encoding($str, $mb_from)) {
  220. // Do the same as //IGNORE with iconv
  221. mb_substitute_character('none');
  222. $out = mb_convert_encoding($str, $mb_to, $mb_from);
  223. mb_substitute_character($mbstring_sch);
  224. if ($out !== false) {
  225. return $out;
  226. }
  227. }
  228. }
  229. }
  230. // convert charset using bundled classes/functions
  231. if ($to == 'UTF-8') {
  232. if ($from == 'UTF7-IMAP') {
  233. if ($_str = self::utf7imap_to_utf8($str)) {
  234. return $_str;
  235. }
  236. }
  237. else if ($from == 'UTF-7') {
  238. if ($_str = self::utf7_to_utf8($str)) {
  239. return $_str;
  240. }
  241. }
  242. else if ($from == 'ISO-8859-1' && function_exists('utf8_encode')) {
  243. return utf8_encode($str);
  244. }
  245. else if (class_exists('utf8')) {
  246. if (!$conv) {
  247. $conv = new utf8($from);
  248. }
  249. else {
  250. $conv->loadCharset($from);
  251. }
  252. if ($_str = $conv->strToUtf8($str)) {
  253. return $_str;
  254. }
  255. }
  256. }
  257. // encode string for output
  258. if ($from == 'UTF-8') {
  259. // @TODO: we need a function for UTF-7 (RFC2152) conversion
  260. if ($to == 'UTF7-IMAP' || $to == 'UTF-7') {
  261. if ($_str = self::utf8_to_utf7imap($str)) {
  262. return $_str;
  263. }
  264. }
  265. else if ($to == 'ISO-8859-1' && function_exists('utf8_decode')) {
  266. return utf8_decode($str);
  267. }
  268. else if (class_exists('utf8')) {
  269. if (!$conv) {
  270. $conv = new utf8($to);
  271. }
  272. else {
  273. $conv->loadCharset($from);
  274. }
  275. if ($_str = $conv->strToUtf8($str)) {
  276. return $_str;
  277. }
  278. }
  279. }
  280. // return original string
  281. return $str;
  282. }
  283. /**
  284. * Converts string from standard UTF-7 (RFC 2152) to UTF-8.
  285. *
  286. * @param string Input string (UTF-7)
  287. *
  288. * @return string Converted string (UTF-8)
  289. */
  290. public static function utf7_to_utf8($str)
  291. {
  292. $Index_64 = array(
  293. 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
  294. 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
  295. 0,0,0,0, 0,0,0,0, 0,0,0,1, 0,0,0,0,
  296. 1,1,1,1, 1,1,1,1, 1,1,0,0, 0,0,0,0,
  297. 0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,
  298. 1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,0,
  299. 0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,
  300. 1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,0,
  301. );
  302. $u7len = strlen($str);
  303. $str = strval($str);
  304. $res = '';
  305. for ($i=0; $u7len > 0; $i++, $u7len--) {
  306. $u7 = $str[$i];
  307. if ($u7 == '+') {
  308. $i++;
  309. $u7len--;
  310. $ch = '';
  311. for (; $u7len > 0; $i++, $u7len--) {
  312. $u7 = $str[$i];
  313. if (!$Index_64[ord($u7)]) {
  314. break;
  315. }
  316. $ch .= $u7;
  317. }
  318. if ($ch == '') {
  319. if ($u7 == '-') {
  320. $res .= '+';
  321. }
  322. continue;
  323. }
  324. $res .= self::utf16_to_utf8(base64_decode($ch));
  325. }
  326. else {
  327. $res .= $u7;
  328. }
  329. }
  330. return $res;
  331. }
  332. /**
  333. * Converts string from UTF-16 to UTF-8 (helper for utf-7 to utf-8 conversion)
  334. *
  335. * @param string Input string
  336. *
  337. * @return string The converted string
  338. */
  339. public static function utf16_to_utf8($str)
  340. {
  341. $len = strlen($str);
  342. $dec = '';
  343. for ($i = 0; $i < $len; $i += 2) {
  344. $c = ord($str[$i]) << 8 | ord($str[$i + 1]);
  345. if ($c >= 0x0001 && $c <= 0x007F) {
  346. $dec .= chr($c);
  347. }
  348. else if ($c > 0x07FF) {
  349. $dec .= chr(0xE0 | (($c >> 12) & 0x0F));
  350. $dec .= chr(0x80 | (($c >> 6) & 0x3F));
  351. $dec .= chr(0x80 | (($c >> 0) & 0x3F));
  352. }
  353. else {
  354. $dec .= chr(0xC0 | (($c >> 6) & 0x1F));
  355. $dec .= chr(0x80 | (($c >> 0) & 0x3F));
  356. }
  357. }
  358. return $dec;
  359. }
  360. /**
  361. * Convert the data ($str) from RFC 2060's UTF-7 to UTF-8.
  362. * If input data is invalid, return the original input string.
  363. * RFC 2060 obviously intends the encoding to be unique (see
  364. * point 5 in section 5.1.3), so we reject any non-canonical
  365. * form, such as &ACY- (instead of &-) or &AMA-&AMA- (instead
  366. * of &AMAAwA-).
  367. *
  368. * Translated from C to PHP by Thomas Bruederli <roundcube@gmail.com>
  369. *
  370. * @param string $str Input string (UTF7-IMAP)
  371. *
  372. * @return string Output string (UTF-8)
  373. */
  374. public static function utf7imap_to_utf8($str)
  375. {
  376. $Index_64 = array(
  377. -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
  378. -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
  379. -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, 63,-1,-1,-1,
  380. 52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1,-1,-1,-1,
  381. -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
  382. 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
  383. -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
  384. 41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1
  385. );
  386. $u7len = strlen($str);
  387. $str = strval($str);
  388. $p = '';
  389. $err = '';
  390. for ($i=0; $u7len > 0; $i++, $u7len--) {
  391. $u7 = $str[$i];
  392. if ($u7 == '&') {
  393. $i++;
  394. $u7len--;
  395. $u7 = $str[$i];
  396. if ($u7len && $u7 == '-') {
  397. $p .= '&';
  398. continue;
  399. }
  400. $ch = 0;
  401. $k = 10;
  402. for (; $u7len > 0; $i++, $u7len--) {
  403. $u7 = $str[$i];
  404. if ((ord($u7) & 0x80) || ($b = $Index_64[ord($u7)]) == -1) {
  405. break;
  406. }
  407. if ($k > 0) {
  408. $ch |= $b << $k;
  409. $k -= 6;
  410. }
  411. else {
  412. $ch |= $b >> (-$k);
  413. if ($ch < 0x80) {
  414. // Printable US-ASCII
  415. if (0x20 <= $ch && $ch < 0x7f) {
  416. return $err;
  417. }
  418. $p .= chr($ch);
  419. }
  420. else if ($ch < 0x800) {
  421. $p .= chr(0xc0 | ($ch >> 6));
  422. $p .= chr(0x80 | ($ch & 0x3f));
  423. }
  424. else {
  425. $p .= chr(0xe0 | ($ch >> 12));
  426. $p .= chr(0x80 | (($ch >> 6) & 0x3f));
  427. $p .= chr(0x80 | ($ch & 0x3f));
  428. }
  429. $ch = ($b << (16 + $k)) & 0xffff;
  430. $k += 10;
  431. }
  432. }
  433. // Non-zero or too many extra bits
  434. if ($ch || $k < 6) {
  435. return $err;
  436. }
  437. // BASE64 not properly terminated
  438. if (!$u7len || $u7 != '-') {
  439. return $err;
  440. }
  441. // Adjacent BASE64 sections
  442. if ($u7len > 2 && $str[$i+1] == '&' && $str[$i+2] != '-') {
  443. return $err;
  444. }
  445. }
  446. // Not printable US-ASCII
  447. else if (ord($u7) < 0x20 || ord($u7) >= 0x7f) {
  448. return $err;
  449. }
  450. else {
  451. $p .= $u7;
  452. }
  453. }
  454. return $p;
  455. }
  456. /**
  457. * Convert the data ($str) from UTF-8 to RFC 2060's UTF-7.
  458. * Unicode characters above U+FFFF are replaced by U+FFFE.
  459. * If input data is invalid, return an empty string.
  460. *
  461. * Translated from C to PHP by Thomas Bruederli <roundcube@gmail.com>
  462. *
  463. * @param string $str Input string (UTF-8)
  464. *
  465. * @return string Output string (UTF7-IMAP)
  466. */
  467. public static function utf8_to_utf7imap($str)
  468. {
  469. $B64Chars = array(
  470. 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
  471. 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
  472. 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
  473. 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
  474. '8', '9', '+', ','
  475. );
  476. $u8len = strlen($str);
  477. $base64 = 0;
  478. $i = 0;
  479. $p = '';
  480. $err = '';
  481. while ($u8len) {
  482. $u8 = $str[$i];
  483. $c = ord($u8);
  484. if ($c < 0x80) {
  485. $ch = $c;
  486. $n = 0;
  487. }
  488. else if ($c < 0xc2) {
  489. return $err;
  490. }
  491. else if ($c < 0xe0) {
  492. $ch = $c & 0x1f;
  493. $n = 1;
  494. }
  495. else if ($c < 0xf0) {
  496. $ch = $c & 0x0f;
  497. $n = 2;
  498. }
  499. else if ($c < 0xf8) {
  500. $ch = $c & 0x07;
  501. $n = 3;
  502. }
  503. else if ($c < 0xfc) {
  504. $ch = $c & 0x03;
  505. $n = 4;
  506. }
  507. else if ($c < 0xfe) {
  508. $ch = $c & 0x01;
  509. $n = 5;
  510. }
  511. else {
  512. return $err;
  513. }
  514. $i++;
  515. $u8len--;
  516. if ($n > $u8len) {
  517. return $err;
  518. }
  519. for ($j=0; $j < $n; $j++) {
  520. $o = ord($str[$i+$j]);
  521. if (($o & 0xc0) != 0x80) {
  522. return $err;
  523. }
  524. $ch = ($ch << 6) | ($o & 0x3f);
  525. }
  526. if ($n > 1 && !($ch >> ($n * 5 + 1))) {
  527. return $err;
  528. }
  529. $i += $n;
  530. $u8len -= $n;
  531. if ($ch < 0x20 || $ch >= 0x7f) {
  532. if (!$base64) {
  533. $p .= '&';
  534. $base64 = 1;
  535. $b = 0;
  536. $k = 10;
  537. }
  538. if ($ch & ~0xffff) {
  539. $ch = 0xfffe;
  540. }
  541. $p .= $B64Chars[($b | $ch >> $k)];
  542. $k -= 6;
  543. for (; $k >= 0; $k -= 6) {
  544. $p .= $B64Chars[(($ch >> $k) & 0x3f)];
  545. }
  546. $b = ($ch << (-$k)) & 0x3f;
  547. $k += 16;
  548. }
  549. else {
  550. if ($base64) {
  551. if ($k > 10) {
  552. $p .= $B64Chars[$b];
  553. }
  554. $p .= '-';
  555. $base64 = 0;
  556. }
  557. $p .= chr($ch);
  558. if (chr($ch) == '&') {
  559. $p .= '-';
  560. }
  561. }
  562. }
  563. if ($base64) {
  564. if ($k > 10) {
  565. $p .= $B64Chars[$b];
  566. }
  567. $p .= '-';
  568. }
  569. return $p;
  570. }
  571. /**
  572. * A method to guess character set of a string.
  573. *
  574. * @param string $string String.
  575. * @param string $failover Default result for failover.
  576. *
  577. * @return string Charset name
  578. */
  579. public static function detect($string, $failover='')
  580. {
  581. if (substr($string, 0, 4) == "\0\0\xFE\xFF") return 'UTF-32BE'; // Big Endian
  582. if (substr($string, 0, 4) == "\xFF\xFE\0\0") return 'UTF-32LE'; // Little Endian
  583. if (substr($string, 0, 2) == "\xFE\xFF") return 'UTF-16BE'; // Big Endian
  584. if (substr($string, 0, 2) == "\xFF\xFE") return 'UTF-16LE'; // Little Endian
  585. if (substr($string, 0, 3) == "\xEF\xBB\xBF") return 'UTF-8';
  586. // heuristics
  587. if ($string[0] == "\0" && $string[1] == "\0" && $string[2] == "\0" && $string[3] != "\0") return 'UTF-32BE';
  588. if ($string[0] != "\0" && $string[1] == "\0" && $string[2] == "\0" && $string[3] == "\0") return 'UTF-32LE';
  589. if ($string[0] == "\0" && $string[1] != "\0" && $string[2] == "\0" && $string[3] != "\0") return 'UTF-16BE';
  590. if ($string[0] != "\0" && $string[1] == "\0" && $string[2] != "\0" && $string[3] == "\0") return 'UTF-16LE';
  591. if (function_exists('mb_detect_encoding')) {
  592. // FIXME: the order is important, because sometimes
  593. // iso string is detected as euc-jp and etc.
  594. $enc = array(
  595. 'UTF-8', 'SJIS', 'GB2312',
  596. 'ISO-8859-1', 'ISO-8859-2', 'ISO-8859-3', 'ISO-8859-4',
  597. 'ISO-8859-5', 'ISO-8859-6', 'ISO-8859-7', 'ISO-8859-8', 'ISO-8859-9',
  598. 'ISO-8859-10', 'ISO-8859-13', 'ISO-8859-14', 'ISO-8859-15', 'ISO-8859-16',
  599. 'WINDOWS-1252', 'WINDOWS-1251', 'EUC-JP', 'EUC-TW', 'KOI8-R', 'BIG5',
  600. 'ISO-2022-KR', 'ISO-2022-JP',
  601. );
  602. $result = mb_detect_encoding($string, join(',', $enc));
  603. }
  604. else {
  605. // No match, check for UTF-8
  606. // from http://w3.org/International/questions/qa-forms-utf-8.html
  607. if (preg_match('/\A(
  608. [\x09\x0A\x0D\x20-\x7E]
  609. | [\xC2-\xDF][\x80-\xBF]
  610. | \xE0[\xA0-\xBF][\x80-\xBF]
  611. | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}
  612. | \xED[\x80-\x9F][\x80-\xBF]
  613. | \xF0[\x90-\xBF][\x80-\xBF]{2}
  614. | [\xF1-\xF3][\x80-\xBF]{3}
  615. | \xF4[\x80-\x8F][\x80-\xBF]{2}
  616. )*\z/xs', substr($string, 0, 2048))
  617. ) {
  618. return 'UTF-8';
  619. }
  620. }
  621. return $result ? $result : $failover;
  622. }
  623. /**
  624. * Removes non-unicode characters from input.
  625. *
  626. * @param mixed $input String or array.
  627. *
  628. * @return mixed String or array
  629. */
  630. public static function clean($input)
  631. {
  632. // handle input of type array
  633. if (is_array($input)) {
  634. foreach ($input as $idx => $val) {
  635. $input[$idx] = self::clean($val);
  636. }
  637. return $input;
  638. }
  639. if (!is_string($input) || $input == '') {
  640. return $input;
  641. }
  642. // iconv/mbstring are much faster (especially with long strings)
  643. if (function_exists('mb_convert_encoding')) {
  644. if (($res = mb_convert_encoding($input, 'UTF-8', 'UTF-8')) !== false) {
  645. return $res;
  646. }
  647. }
  648. if (function_exists('iconv')) {
  649. if (($res = @iconv('UTF-8', 'UTF-8//IGNORE', $input)) !== false) {
  650. return $res;
  651. }
  652. }
  653. $seq = '';
  654. $out = '';
  655. $regexp = '/^('.
  656. // '[\x00-\x7F]'. // UTF8-1
  657. '|[\xC2-\xDF][\x80-\xBF]'. // UTF8-2
  658. '|\xE0[\xA0-\xBF][\x80-\xBF]'. // UTF8-3
  659. '|[\xE1-\xEC][\x80-\xBF][\x80-\xBF]'. // UTF8-3
  660. '|\xED[\x80-\x9F][\x80-\xBF]'. // UTF8-3
  661. '|[\xEE-\xEF][\x80-\xBF][\x80-\xBF]'. // UTF8-3
  662. '|\xF0[\x90-\xBF][\x80-\xBF][\x80-\xBF]'. // UTF8-4
  663. '|[\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF]'.// UTF8-4
  664. '|\xF4[\x80-\x8F][\x80-\xBF][\x80-\xBF]'. // UTF8-4
  665. ')$/';
  666. for ($i = 0, $len = strlen($input); $i < $len; $i++) {
  667. $chr = $input[$i];
  668. $ord = ord($chr);
  669. // 1-byte character
  670. if ($ord <= 0x7F) {
  671. if ($seq) {
  672. $out .= preg_match($regexp, $seq) ? $seq : '';
  673. }
  674. $seq = '';
  675. $out .= $chr;
  676. // first (or second) byte of multibyte sequence
  677. }
  678. else if ($ord >= 0xC0) {
  679. if (strlen($seq) > 1) {
  680. $out .= preg_match($regexp, $seq) ? $seq : '';
  681. $seq = '';
  682. }
  683. else if ($seq && ord($seq) < 0xC0) {
  684. $seq = '';
  685. }
  686. $seq .= $chr;
  687. // next byte of multibyte sequence
  688. }
  689. else if ($seq) {
  690. $seq .= $chr;
  691. }
  692. }
  693. if ($seq) {
  694. $out .= preg_match($regexp, $seq) ? $seq : '';
  695. }
  696. return $out;
  697. }
  698. }