PageRenderTime 42ms CodeModel.GetById 19ms RepoModel.GetById 0ms app.codeStats 0ms

/include/utf8/utils/bad.php

https://bitbucket.org/gencer/fluxbb
PHP | 430 lines | 216 code | 47 blank | 167 comment | 63 complexity | eb0a7cb5d6dd7c6aec64c374f09bd917 MD5 | raw file
Possible License(s): GPL-2.0
  1. <?php
  2. /**
  3. * @version $Id: bad.php,v 1.2 2006/02/26 13:20:44 harryf Exp $
  4. * Tools for locating / replacing bad bytes in UTF-8 strings
  5. * The Original Code is Mozilla Communicator client code.
  6. * The Initial Developer of the Original Code is
  7. * Netscape Communications Corporation.
  8. * Portions created by the Initial Developer are Copyright (C) 1998
  9. * the Initial Developer. All Rights Reserved.
  10. * Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
  11. * Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
  12. * @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
  13. * @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
  14. * @see http://hsivonen.iki.fi/php-utf8/
  15. * @package utf8
  16. * @subpackage bad
  17. * @see utf8_is_valid
  18. */
  19. /**
  20. * Locates the first bad byte in a UTF-8 string returning it's
  21. * byte index in the string
  22. * PCRE Pattern to locate bad bytes in a UTF-8 string
  23. * Comes from W3 FAQ: Multilingual Forms
  24. * Note: modified to include full ASCII range including control chars
  25. * @see http://www.w3.org/International/questions/qa-forms-utf-8
  26. * @param string
  27. * @return mixed integer byte index or FALSE if no bad found
  28. * @package utf8
  29. * @subpackage bad
  30. */
  31. function utf8_bad_find($str)
  32. {
  33. $UTF8_BAD =
  34. '([\x00-\x7F]'. # ASCII (including control chars)
  35. '|[\xC2-\xDF][\x80-\xBF]'. # Non-overlong 2-byte
  36. '|\xE0[\xA0-\xBF][\x80-\xBF]'. # Excluding overlongs
  37. '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # Straight 3-byte
  38. '|\xED[\x80-\x9F][\x80-\xBF]'. # Excluding surrogates
  39. '|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # Planes 1-3
  40. '|[\xF1-\xF3][\x80-\xBF]{3}'. # Planes 4-15
  41. '|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # Plane 16
  42. '|(.{1}))'; # Invalid byte
  43. $pos = 0;
  44. $badList = array();
  45. while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches))
  46. {
  47. $bytes = strlen($matches[0]);
  48. if (isset($matches[2]))
  49. return $pos;
  50. $pos += $bytes;
  51. $str = substr($str,$bytes);
  52. }
  53. return false;
  54. }
  55. /**
  56. * Locates all bad bytes in a UTF-8 string and returns a list of their
  57. * byte index in the string
  58. * PCRE Pattern to locate bad bytes in a UTF-8 string
  59. * Comes from W3 FAQ: Multilingual Forms
  60. * Note: modified to include full ASCII range including control chars
  61. * @see http://www.w3.org/International/questions/qa-forms-utf-8
  62. * @param string
  63. * @return mixed array of integers or FALSE if no bad found
  64. * @package utf8
  65. * @subpackage bad
  66. */
  67. function utf8_bad_findall($str)
  68. {
  69. $UTF8_BAD =
  70. '([\x00-\x7F]'. # ASCII (including control chars)
  71. '|[\xC2-\xDF][\x80-\xBF]'. # Non-overlong 2-byte
  72. '|\xE0[\xA0-\xBF][\x80-\xBF]'. # Excluding overlongs
  73. '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # Straight 3-byte
  74. '|\xED[\x80-\x9F][\x80-\xBF]'. # Excluding surrogates
  75. '|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # Planes 1-3
  76. '|[\xF1-\xF3][\x80-\xBF]{3}'. # Planes 4-15
  77. '|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # Plane 16
  78. '|(.{1}))'; # Invalid byte
  79. $pos = 0;
  80. $badList = array();
  81. while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches))
  82. {
  83. $bytes = strlen($matches[0]);
  84. if (isset($matches[2]))
  85. $badList[] = $pos;
  86. $pos += $bytes;
  87. $str = substr($str,$bytes);
  88. }
  89. if (count($badList) > 0)
  90. return $badList;
  91. return false;
  92. }
  93. /**
  94. * Strips out any bad bytes from a UTF-8 string and returns the rest
  95. * PCRE Pattern to locate bad bytes in a UTF-8 string
  96. * Comes from W3 FAQ: Multilingual Forms
  97. * Note: modified to include full ASCII range including control chars
  98. * @see http://www.w3.org/International/questions/qa-forms-utf-8
  99. * @param string
  100. * @return string
  101. * @package utf8
  102. * @subpackage bad
  103. */
  104. function utf8_bad_strip($original)
  105. {
  106. return utf8_bad_replace($original, '');
  107. }
  108. /**
  109. * Replace bad bytes with an alternative character - ASCII character
  110. * recommended is replacement char
  111. * PCRE Pattern to locate bad bytes in a UTF-8 string
  112. * Comes from W3 FAQ: Multilingual Forms
  113. * Note: modified to include full ASCII range including control chars
  114. * @see http://www.w3.org/International/questions/qa-forms-utf-8
  115. * @param string to search
  116. * @param string to replace bad bytes with (defaults to '?') - use ASCII
  117. * @return string
  118. * @package utf8
  119. * @subpackage bad
  120. */
  121. function utf8_bad_replace($original, $replace = '?') {
  122. $result = '';
  123. $strlen = strlen($original);
  124. for ($i = 0; $i < $strlen;) {
  125. $char = $original[$i++];
  126. $byte = ord($char);
  127. if ($byte < 0x80) $bytes = 0; // 1-bytes (00000000 - 01111111)
  128. else if ($byte < 0xC0) { // 1-bytes (10000000 - 10111111)
  129. $result .= $replace;
  130. continue;
  131. }
  132. else if ($byte < 0xE0) $bytes = 1; // 2-bytes (11000000 - 11011111)
  133. else if ($byte < 0xF0) $bytes = 2; // 3-bytes (11100000 - 11101111)
  134. else if ($byte < 0xF8) $bytes = 3; // 4-bytes (11110000 - 11110111)
  135. else if ($byte < 0xFC) $bytes = 4; // 5-bytes (11111000 - 11111011)
  136. else if ($byte < 0xFE) $bytes = 5; // 6-bytes (11111100 - 11111101)
  137. else { // Otherwise it's something invalid
  138. $result .= $replace;
  139. continue;
  140. }
  141. // Check our input actually has enough data
  142. if ($i + $bytes > $strlen) {
  143. $result .= $replace;
  144. continue;
  145. }
  146. // If we've got this far then we have a multiple-byte character
  147. for ($j = 0; $j < $bytes; $j++) {
  148. $byte = $original[$i + $j];
  149. $char .= $byte;
  150. $byte = ord($byte);
  151. // Every following byte must be 10000000 - 10111111
  152. if ($byte < 0x80 || $byte > 0xBF) {
  153. $result .= $replace;
  154. continue 2;
  155. }
  156. }
  157. $i += $bytes;
  158. $result .= $char;
  159. }
  160. return $result;
  161. }
  162. /**
  163. * Return code from utf8_bad_identify() when a five octet sequence is detected.
  164. * Note: 5 octets sequences are valid UTF-8 but are not supported by Unicode so
  165. * do not represent a useful character
  166. * @see utf8_bad_identify
  167. * @package utf8
  168. * @subpackage bad
  169. */
  170. define('UTF8_BAD_5OCTET', 1);
  171. /**
  172. * Return code from utf8_bad_identify() when a six octet sequence is detected.
  173. * Note: 6 octets sequences are valid UTF-8 but are not supported by Unicode so
  174. * do not represent a useful character
  175. * @see utf8_bad_identify
  176. * @package utf8
  177. * @subpackage bad
  178. */
  179. define('UTF8_BAD_6OCTET', 2);
  180. /**
  181. * Return code from utf8_bad_identify().
  182. * Invalid octet for use as start of multi-byte UTF-8 sequence
  183. * @see utf8_bad_identify
  184. * @package utf8
  185. * @subpackage bad
  186. */
  187. define('UTF8_BAD_SEQID', 3);
  188. /**
  189. * Return code from utf8_bad_identify().
  190. * From Unicode 3.1, non-shortest form is illegal
  191. * @see utf8_bad_identify
  192. * @package utf8
  193. * @subpackage bad
  194. */
  195. define('UTF8_BAD_NONSHORT', 4);
  196. /**
  197. * Return code from utf8_bad_identify().
  198. * From Unicode 3.2, surrogate characters are illegal
  199. * @see utf8_bad_identify
  200. * @package utf8
  201. * @subpackage bad
  202. */
  203. define('UTF8_BAD_SURROGATE', 5);
  204. /**
  205. * Return code from utf8_bad_identify().
  206. * Codepoints outside the Unicode range are illegal
  207. * @see utf8_bad_identify
  208. * @package utf8
  209. * @subpackage bad
  210. */
  211. define('UTF8_BAD_UNIOUTRANGE', 6);
  212. /**
  213. * Return code from utf8_bad_identify().
  214. * Incomplete multi-octet sequence
  215. * Note: this is kind of a "catch-all"
  216. * @see utf8_bad_identify
  217. * @package utf8
  218. * @subpackage bad
  219. */
  220. define('UTF8_BAD_SEQINCOMPLETE', 7);
  221. /**
  222. * Reports on the type of bad byte found in a UTF-8 string. Returns a
  223. * status code on the first bad byte found
  224. * @author <hsivonen@iki.fi>
  225. * @param string UTF-8 encoded string
  226. * @return mixed integer constant describing problem or FALSE if valid UTF-8
  227. * @see utf8_bad_explain
  228. * @see http://hsivonen.iki.fi/php-utf8/
  229. * @package utf8
  230. * @subpackage bad
  231. */
  232. function utf8_bad_identify($str, &$i)
  233. {
  234. $mState = 0; // Cached expected number of octets after the current octet
  235. // until the beginning of the next UTF8 character sequence
  236. $mUcs4 = 0; // Cached Unicode character
  237. $mBytes = 1; // Cached expected number of octets in the current sequence
  238. $len = strlen($str);
  239. for($i=0; $i < $len; $i++)
  240. {
  241. $in = ord($str{$i});
  242. if ( $mState == 0)
  243. {
  244. // When mState is zero we expect either a US-ASCII character or a multi-octet sequence.
  245. if (0 == (0x80 & ($in)))
  246. {
  247. // US-ASCII, pass straight through.
  248. $mBytes = 1;
  249. }
  250. else if (0xC0 == (0xE0 & ($in)))
  251. {
  252. // First octet of 2 octet sequence
  253. $mUcs4 = ($in);
  254. $mUcs4 = ($mUcs4 & 0x1F) << 6;
  255. $mState = 1;
  256. $mBytes = 2;
  257. }
  258. else if (0xE0 == (0xF0 & ($in)))
  259. {
  260. // First octet of 3 octet sequence
  261. $mUcs4 = ($in);
  262. $mUcs4 = ($mUcs4 & 0x0F) << 12;
  263. $mState = 2;
  264. $mBytes = 3;
  265. }
  266. else if (0xF0 == (0xF8 & ($in)))
  267. {
  268. // First octet of 4 octet sequence
  269. $mUcs4 = ($in);
  270. $mUcs4 = ($mUcs4 & 0x07) << 18;
  271. $mState = 3;
  272. $mBytes = 4;
  273. }
  274. else if (0xF8 == (0xFC & ($in)))
  275. {
  276. /* First octet of 5 octet sequence.
  277. *
  278. * This is illegal because the encoded codepoint must be either
  279. * (a) not the shortest form or
  280. * (b) outside the Unicode range of 0-0x10FFFF.
  281. */
  282. return UTF8_BAD_5OCTET;
  283. }
  284. else if (0xFC == (0xFE & ($in)))
  285. {
  286. // First octet of 6 octet sequence, see comments for 5 octet sequence.
  287. return UTF8_BAD_6OCTET;
  288. }
  289. else
  290. {
  291. // Current octet is neither in the US-ASCII range nor a legal first
  292. // octet of a multi-octet sequence.
  293. return UTF8_BAD_SEQID;
  294. }
  295. }
  296. else
  297. {
  298. // When mState is non-zero, we expect a continuation of the multi-octet sequence
  299. if (0x80 == (0xC0 & ($in)))
  300. {
  301. // Legal continuation.
  302. $shift = ($mState - 1) * 6;
  303. $tmp = $in;
  304. $tmp = ($tmp & 0x0000003F) << $shift;
  305. $mUcs4 |= $tmp;
  306. /**
  307. * End of the multi-octet sequence. mUcs4 now contains the final
  308. * Unicode codepoint to be output
  309. */
  310. if (0 == --$mState)
  311. {
  312. // From Unicode 3.1, non-shortest form is illegal
  313. if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
  314. ((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
  315. ((4 == $mBytes) && ($mUcs4 < 0x10000)) )
  316. return UTF8_BAD_NONSHORT;
  317. else if (($mUcs4 & 0xFFFFF800) == 0xD800) // From Unicode 3.2, surrogate characters are illegal
  318. return UTF8_BAD_SURROGATE;
  319. else if ($mUcs4 > 0x10FFFF) // Codepoints outside the Unicode range are illegal
  320. return UTF8_BAD_UNIOUTRANGE;
  321. // Initialize UTF8 cache
  322. $mState = 0;
  323. $mUcs4 = 0;
  324. $mBytes = 1;
  325. }
  326. }
  327. else
  328. {
  329. // ((0xC0 & (*in) != 0x80) && (mState != 0))
  330. // Incomplete multi-octet sequence.
  331. $i--;
  332. return UTF8_BAD_SEQINCOMPLETE;
  333. }
  334. }
  335. }
  336. // Incomplete multi-octet sequence
  337. if ($mState != 0)
  338. {
  339. $i--;
  340. return UTF8_BAD_SEQINCOMPLETE;
  341. }
  342. // No bad octets found
  343. $i = null;
  344. return false;
  345. }
  346. /**
  347. * Takes a return code from utf8_bad_identify() are returns a message
  348. * (in English) explaining what the problem is.
  349. * @param int return code from utf8_bad_identify
  350. * @return mixed string message or FALSE if return code unknown
  351. * @see utf8_bad_identify
  352. * @package utf8
  353. * @subpackage bad
  354. */
  355. function utf8_bad_explain($code)
  356. {
  357. switch ($code)
  358. {
  359. case UTF8_BAD_5OCTET:
  360. return 'Five octet sequences are valid UTF-8 but are not supported by Unicode';
  361. break;
  362. case UTF8_BAD_6OCTET:
  363. return 'Six octet sequences are valid UTF-8 but are not supported by Unicode';
  364. break;
  365. case UTF8_BAD_SEQID:
  366. return 'Invalid octet for use as start of multi-byte UTF-8 sequence';
  367. break;
  368. case UTF8_BAD_NONSHORT:
  369. return 'From Unicode 3.1, non-shortest form is illegal';
  370. break;
  371. case UTF8_BAD_SURROGATE:
  372. return 'From Unicode 3.2, surrogate characters are illegal';
  373. break;
  374. case UTF8_BAD_UNIOUTRANGE:
  375. return 'Codepoints outside the Unicode range are illegal';
  376. break;
  377. case UTF8_BAD_SEQINCOMPLETE:
  378. return 'Incomplete multi-octet sequence';
  379. break;
  380. }
  381. trigger_error('Unknown error code: '.$code, E_USER_WARNING);
  382. return false;
  383. }