/phpBB/develop/generate_utf_confusables.php
PHP | 245 lines | 178 code | 26 blank | 41 comment | 23 complexity | 8ac44e08128d2a54e35baa16f45a824b MD5 | raw file
- <?php
- /**
- *
- * This file is part of the phpBB Forum Software package.
- *
- * @copyright (c) phpBB Limited <https://www.phpbb.com>
- * @license GNU General Public License, version 2 (GPL-2.0)
- *
- * For full copyright and license information, please see
- * the docs/CREDITS.txt file.
- *
- */
- if (php_sapi_name() != 'cli')
- {
- die("This program must be run from the command line.\n");
- }
- //
- // Security message:
- //
- // This script is potentially dangerous.
- // Remove or comment the next line (die(".... ) to enable this script.
- // Do NOT FORGET to either remove this script or disable it after you have used it.
- //
- die("Please read the first lines of this script for instructions on how to enable it");
- set_time_limit(0);
- define('IN_PHPBB', true);
- $phpbb_root_path = '../';
- $phpEx = substr(strrchr(__FILE__, '.'), 1);
- echo "Checking for required files\n";
- download('http://unicode.org/reports/tr39/data/confusables.txt');
- download('http://unicode.org/Public/UNIDATA/CaseFolding.txt');
- echo "\n";
- /**
- * Load the confusables table
- */
- echo "Loading confusables\n";
- $unidata = file_get_contents('confusables.txt');
- /**
- * Load the CaseFolding table
- */
- echo "Loading CaseFolding\n";
- $casefolds = file_get_contents('CaseFolding.txt');
- function utf8_chr($cp)
- {
- if ($cp > 0xFFFF)
- {
- return chr(0xF0 | ($cp >> 18)) . chr(0x80 | (($cp >> 12) & 0x3F)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
- }
- else if ($cp > 0x7FF)
- {
- return chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F));
- }
- else if ($cp > 0x7F)
- {
- return chr(0xC0 | ($cp >> 6)) . chr(0x80 | ($cp & 0x3F));
- }
- else
- {
- return chr($cp);
- }
- }
- preg_match_all('/^([0-9A-F]+) ;\s((?:[0-9A-F]+ )*);.*?$/im', $unidata, $array, PREG_SET_ORDER);
- preg_match_all('/^([0-9A-F]+); ([CFS]); ([0-9A-F]+(?: [0-9A-F]+)*);/im', $casefolds, $casefold_array);
- // some that we defined ourselves
- $uniarray = array(
- "\xC2\xA1" => "\x69", // EXCLAMATION MARK, INVERTED => LATIN SMALL LETTER I
- "\xC7\x83" => "\x21", // LATIN LETTER RETROFLEX CLICK => EXCLAMATION MARK
- "\xCE\xB1" => "\x61", // GREEK SMALL LETTER ALPHA => LATIN SMALL LETTER A
- "\xE1\x9A\x80" => "\x20", // OGHAM SPACE MARK
- "\xC2\xAD" => '', // HYPHEN, SOFT => empty string
- "\xDB\x9D" => '', // ARABIC END OF AYAH
- "\xDC\x8F" => '', // SYRIAC ABBREVIATION MARK
- "\xE1\xA0\x86" => '', // MONGOLIAN TODO SOFT HYPHEN
- "\xE1\xA0\x8E" => '', // MONGOLIAN VOWEL SEPARATOR
- "\xE2\x80\x8B" => '', // ZERO WIDTH SPACE
- "\xE2\x80\x8C" => '', // ZERO WIDTH NON-JOINER
- "\xE2\x80\x8D" => '', // ZERO WIDTH JOINER
- "\xE2\x80\xA8" => '', // LINE SEPARATOR
- "\xE2\x80\xA9" => '', // PARAGRAPH SEPARATOR
- "\xE2\x81\xA0" => '', // WORD JOINER
- "\xE2\x81\xA1" => '', // FUNCTION APPLICATION
- "\xE2\x81\xA2" => '', // INVISIBLE TIMES
- "\xE2\x81\xA3" => '', // INVISIBLE SEPARATOR
- "\xE2\x81\xAA" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAB" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAC" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAD" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAE" => '', // [CONTROL CHARACTERS]
- "\xE2\x81\xAF" => '', // [CONTROL CHARACTERS]
- "\xEF\xBB\xBF" => '', // ZERO WIDTH NO-BREAK SPACE
- "\xEF\xBF\xB9" => '', // [CONTROL CHARACTERS]
- "\xEF\xBF\xBA" => '', // [CONTROL CHARACTERS]
- "\xEF\xBF\xBB" => '', // [CONTROL CHARACTERS]
- "\xEF\xBF\xBC" => '', // [CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB3" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB4" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB5" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB6" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB7" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB8" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xB9" => '', // [MUSICAL CONTROL CHARACTERS]
- "\xF0\x9D\x85\xBA" => '', // [MUSICAL CONTROL CHARACTERS]
- );
- $copy = $uniarray;
- /**
- * @todo we need to check that the $uniarray does not reverse any of the mappings defined in the unicode definition
- */
- foreach ($array as $value)
- {
- $temp_hold = implode(array_map('utf8_chr', array_map('hexdec', explode(' ', trim($value[2])))));
- if (isset($copy[utf8_chr(hexdec((string)$value[1]))]))
- {
- $num = '';
- $string = utf8_chr(hexdec((string)$value[1]));
- for ($i = 0; $i < strlen($string); $i++)
- {
- $num .= '\x' . str_pad(base_convert(ord($string[$i]), 10, 16), 2, '0', STR_PAD_LEFT);
- }
- echo $num . "\n";
- if ($uniarray[$string] != $temp_hold)
- {
- echo " --> $string\n";
- echo " --> " . $temp_hold . "\n";
- }
- }
- // do some tests for things that transform into something with the number one
- if (strpos($temp_hold, utf8_chr(0x0031)) !== false)
- {
- // any kind of letter L?
- if (strpos($value[0], 'LETTER L') !== false || strpos($value[0], 'IOTA') !== false || strpos($value[0], 'SMALL L ') !== false || preg_match('/SMALL LIGATURE [^L]*L /', $value[0]))
- {
- // replace all of the mappings that transform some sort of letter l to number one instead to some sort of letter l to latin small letter l
- $temp_hold = str_replace(utf8_chr(0x0031), utf8_chr(0x006C), $temp_hold);
- }
- }
- // uppercased chars that were folded do not exist in this universe,
- // no amount of normalization could ever "trick" this into not working
- if (in_array($value[1], $casefold_array[1]))
- {
- continue;
- }
- $uniarray[utf8_chr(hexdec((string)$value[1]))] = $temp_hold;
- }
- echo "Writing to confusables.$phpEx\n";
- $fp = fopen($phpbb_root_path . 'includes/utf/data/confusables.' . $phpEx, 'wb');
- fwrite($fp, '<?php return ' . my_var_export($uniarray) . ';');
- fclose($fp);
- /**
- * Return a parsable string representation of a variable
- *
- * This is function is limited to array/strings/integers
- *
- * @param mixed $var Variable
- * @return string PHP code representing the variable
- */
- function my_var_export($var)
- {
- if (is_array($var))
- {
- $lines = array();
- foreach ($var as $k => $v)
- {
- $lines[] = my_var_export($k) . '=>' . my_var_export($v);
- }
- return 'array(' . implode(',', $lines) . ')';
- }
- else if (is_string($var))
- {
- return "'" . str_replace(array('\\', "'"), array('\\\\', "\\'"), $var) . "'";
- }
- else
- {
- return $var;
- }
- }
- /**
- * Download a file to the develop/ dir
- *
- * @param string $url URL of the file to download
- * @return null
- */
- function download($url)
- {
- global $phpbb_root_path;
- if (file_exists($phpbb_root_path . 'develop/' . basename($url)))
- {
- return;
- }
- echo 'Downloading from ', $url, ' ';
- if (!$fpr = fopen($url, 'rb'))
- {
- die("Can't download from $url\nPlease download it yourself and put it in the develop/ dir, kthxbai");
- }
- if (!$fpw = fopen($phpbb_root_path . 'develop/' . basename($url), 'wb'))
- {
- die("Can't open develop/" . basename($url) . " for output... please check your permissions or something");
- }
- $i = 0;
- $chunk = 32768;
- $done = '';
- while (!feof($fpr))
- {
- $i += fwrite($fpw, fread($fpr, $chunk));
- echo str_repeat("\x08", strlen($done));
- $done = ($i >> 10) . ' KiB';
- echo $done;
- }
- fclose($fpr);
- fclose($fpw);
- echo "\n";
- }