PageRenderTime 77ms CodeModel.GetById 30ms RepoModel.GetById 1ms app.codeStats 0ms

/search/inc/spider.inc.php

https://bitbucket.org/molusc/sma-website
PHP | 2239 lines | 1773 code | 272 blank | 194 comment | 438 complexity | 8db91d84a3e69e7c0759b1218c936bc1 MD5 | raw file
Possible License(s): BSD-3-Clause 
    

  1. <?php

    2. 

  2. 

    3. 

  3. /******************************************************************************

    4. 

  4.  * iSearch2 - website search engine                                           *

    5. 

  5.  *                                                                            *

    6. 

  6.  * Visit the iSearch homepage at http://www.iSearchTheNet/isearch             *

    7. 

  7.  *                                                                            *

    8. 

  8.  * Copyright (C) 2002-2007 Z-Host. All rights reserved.                       *

    9. 

  9.  *                                                                            *

    10. 

  10.  ******************************************************************************/

    11. 

  11. 

    12. 

  12. if ( !defined('IN_ISEARCH') )

    13. 

  13. {

    14. 

  14.     die('Hacking attempt');

    15. 

  15. }

    16. 

  16. 

    17. 

  17. /* Parse the robots.txt stored in the database */

    18. 

  18. function isearch_parseRobots($domain)

    19. 

  19. {

    20. 

  20.     global $isearch_config;

    21. 

  21.     global $isearch_table_info;

    22. 

  22.     global $isearch_db;

    23. 

  23.     global $isearch_base;

    24. 

  24. 

    25. 

  25.     $isearch_config['robots_domains'][] = $domain;

    26. 

  26. 

    27. 

  27.     // Store in a temp veriable to allow other relative URLs to be evaluated

    28. 

  28.     // after parsing robots.txt.

    29. 

  29.     $isearch_base_tmp = $isearch_base;

    30. 

  30.     $allData = isearch_readFile("http://$domain/robots.txt");

    31. 

  31.     $isearch_base = $isearch_base_tmp;

    32. 

  32. 

    33. 

  33.     if ($allData != '')

    34. 

  34.     {

    35. 

  35.         $allData = strtolower($allData);

    36. 

  36.         $lines = split("(\r|\n)", $allData);

    37. 

  37.         $validUseragent = False;

    38. 

  38.         $matched = False;

    39. 

  39. 

    40. 

  40.         foreach ($lines as $line)

    41. 

  41.         {

    42. 

  42.             $line = ereg_replace('#.*$', '', $line);

    43. 

  43.             $line = ereg_replace('[[:space:]]+', ' ', $line);

    44. 

  44.             $temp = explode(':', $line, 2);

    45. 

  45.             if (count($temp) == 2)

    46. 

  46.             {

    47. 

  47.                 $field = trim($temp[0]);

    48. 

  48.                 $value = trim($temp[1]);

    49. 

  49.                 if ($field == 'user-agent')

    50. 

  50.                 {

    51. 

  51.                     $validUseragent = False;

    52. 

  52.                     $useragents = explode(' ', $value);

    53. 

  53.                     foreach ($useragents as $useragent)

    54. 

  54.                     {

    55. 

  55.                         if (($useragent == 'isearch') || (($useragent == '*') && (!$matched)))

    56. 

  56.                         {

    57. 

  57.                             $matched = True;

    58. 

  58.                             $validUseragent = True;

    59. 

  59.                         }

    60. 

  60.                     }

    61. 

  61.                 }

    62. 

  62.                 else if (($validUseragent) && ($field == 'disallow'))

    63. 

  63.                 {

    64. 

  64.                     if ($value == '')

    65. 

  65.                     {

    66. 

  66.                         /* This is an allow - remove all previous disallows */

    67. 

  67.                         unset($disallow);

    68. 

  68.                     }

    69. 

  69.                     else

    70. 

  70.                     {

    71. 

  71.                         $disallow[] = $value;

    72. 

  72.                     }

    73. 

  73.                 }

    74. 

  74.             }

    75. 

  75.         }

    76. 

  76. 

    77. 

  77.         if (isset($disallow))

    78. 

  78.         {

    79. 

  79.             foreach ($disallow as $temp)

    80. 

  80.             {

    81. 

  81.                 if ($temp{0} != '/')

    82. 

  82.                 {

    83. 

  83.                     $temp = '/' . $temp;

    84. 

  84.                 }

    85. 

  85.                 $url = "^http://$domain$temp";

    86. 

  86.                 $url = ereg_replace('\.', '\.', $url);

    87. 

  87.                 $url = ereg_replace('\*', '.*', $url);

    88. 

  88.                 $url = ereg_replace('\?', '\?', $url);

    89. 

  89.                 $url = ereg_replace('\+', '\+', $url);

    90. 

  90.                 $isearch_config['robots_excludes'][] = $url;

    91. 

  91.             }

    92. 

  92.         }

    93. 

  93.     }

    94. 

  94. 

    95. 

  95.     if (!mysql_query("UPDATE $isearch_table_info SET robots_domains='" . isearch_escape_string(implode(" ", $isearch_config['robots_domains'])) . "', robots_excludes='" . isearch_escape_string(implode(" ", $isearch_config['robots_excludes'])) . "' WHERE id='1'", $isearch_db))

    96. 

  96.     {

    97. 

  97.         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    98. 

  98.     }

    99. 

  99. }

    100. 

  100. 

    101. 

  101. 

    102. 

  102. /* Clear the iSearch log file */

    103. 

  103. function isearch_clearLog()

    104. 

  104. {

    105. 

  105.     global $isearch_table_spider_log;

    106. 

  106.     global $isearch_db;

    107. 

  107. 

    108. 

  108.     mysql_query("DELETE FROM $isearch_table_spider_log", $isearch_db);

    109. 

  109. }

    110. 

  110. 

    111. 

  111. 

    112. 

  112. /* Return the contents of the spider log */

    113. 

  113. function isearch_getLog()

    114. 

  114. {

    115. 

  115.     global $isearch_table_spider_log;

    116. 

  116.     global $isearch_db;

    117. 

  117. 

    118. 

  118.     $log = '';

    119. 

  119. 

    120. 

  120.     $result = mysql_query("SELECT * FROM $isearch_table_spider_log ORDER BY id", $isearch_db);

    121. 

  121.     if ($result)

    122. 

  122.     {

    123. 

  123.         while ($item = mysql_fetch_object($result))

    124. 

  124.         {

    125. 

  125.             $log .= str_replace(' ', '&nbsp;', htmlentities($item->msg)) . "<BR>\n";

    126. 

  126.         }

    127. 

  127.     }

    128. 

  128. 

    129. 

  129.     return $log;

    130. 

  130. }

    131. 

  131. 

    132. 

  132. 

    133. 

  133. /* Save the string in the iSearch log file */

    134. 

  134. function isearch_log($string, $level=1)

    135. 

  135. {

    136. 

  136.     global $isearch_table_spider_log;

    137. 

  137.     global $isearch_db;

    138. 

  138.     global $isearch_config;

    139. 

  139.     global $isearch_fromCommandLine;

    140. 

  140.     global $isearch_logEchoLevel;

    141. 

  141. 

    142. 

  142.     if ($level <= $isearch_config['log_level'])

    143. 

  143.     {

    144. 

  144.         mysql_query("INSERT INTO $isearch_table_spider_log (msg) VALUES ('" . isearch_escape_string($string) . "')", $isearch_db);

    145. 

  145.     }

    146. 

  146. 

    147. 

  147.     if (isset($isearch_fromCommandLine) && $isearch_fromCommandLine)

    148. 

  148.     {

    149. 

  149.         if ($level <= $isearch_logEchoLevel)

    150. 

  150.         {

    151. 

  151.             echo "$string\n";

    152. 

  152.         }

    153. 

  153.     }

    154. 

  154.     else

    155. 

  155.     {

    156. 

  156.         if ($level <= $isearch_config['log_echo_level'])

    157. 

  157.         {

    158. 

  158.             echo str_replace(' ', '&nbsp;', htmlentities($string)) . "<BR>\n";

    159. 

  159.         }

    160. 

  160.     }

    161. 

  161. }

    162. 

  162. 

    163. 

  163. 

    164. 

  164. /* Clean up a string to make it suitable for storing in search index */

    165. 

  165. function isearch_cleanString($data, $charset)

    166. 

  166. {

    167. 

  167.     global $isearch_config;

    168. 

  168. 

    169. 

  169.     if ($isearch_config['char_set_8_bit'])

    170. 

  170.     {

    171. 

  171.         /* Convert to lower case, doing accented character conversion correctly */

    172. 

  172.         $data = strtr($data, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ?ÁÂ?Ä??Ç?É?Ë?ÍÎ????ÓÔ?Ö×??Ú?ÜÝ?' . chr(0x8a) . chr(0x8e) ,

    173. 

  173.                              'abcdefghijklmnopqrstuvwxyz?áâ?ä??ç?é?ë?íî????óô?ö÷??ú?üý?' . chr(0x9a) . chr(0x9e) );

    174. 

  174.     }

    175. 

  175. 

    176. 

  176.     /* Strip out all HTML tags */

    177. 

  177.     $data = strip_tags($data);

    178. 

  178. 

    179. 

  179.     /* Replace some breaking chars with spaces */

    180. 

  180.     $data = ereg_replace('[\\;\?!]+', ' ', $data);

    181. 

  181. 

    182. 

  182.     if ($isearch_config['allow_dashes'] == 0)

    183. 

  183.     {

    184. 

  184.         /* Replace dashes with spaces */

    185. 

  185.         $data = str_replace('-', ' ', $data);

    186. 

  186.     }

    187. 

  187.     else if ($isearch_config['allow_dashes'] == 1)

    188. 

  188.     {

    189. 

  189.         /* Allow within words */

    190. 

  190.         $data = str_replace(' -', ' ', str_replace('- ', ' ', $data));

    191. 

  191.     }

    192. 

  192.     else if ($isearch_config['allow_dashes'] == 3)

    193. 

  193.     {

    194. 

  194.         /* Remove All */

    195. 

  195.         $data = str_replace('-', '', $data);

    196. 

  196.     }

    197. 

  197. 

    198. 

  198.     if ($isearch_config['allow_colons'] == 0)

    199. 

  199.     {

    200. 

  200.         /* Replace with spaces */

    201. 

  201.         $data = str_replace(':', ' ', $data);

    202. 

  202.     }

    203. 

  203.     else if ($isearch_config['allow_colons'] == 1)

    204. 

  204.     {

    205. 

  205.         /* Allow within words */

    206. 

  206.         $data = str_replace(' :', ' ', str_replace(': ', ' ', $data));

    207. 

  207.     }

    208. 

  208.     else if ($isearch_config['allow_colons'] == 3)

    209. 

  209.     {

    210. 

  210.         /* Remove All */

    211. 

  211.         $data = str_replace(':', '', $data);

    212. 

  212.     }

    213. 

  213. 

    214. 

  214.     if ($isearch_config['allow_dots'] == 0)

    215. 

  215.     {

    216. 

  216.         /* Replace with spaces */

    217. 

  217.         $data = str_replace('.', ' ', $data);

    218. 

  218.     }

    219. 

  219.     else if ($isearch_config['allow_dots'] == 1)

    220. 

  220.     {

    221. 

  221.         /* Allow within words */

    222. 

  222.         $data = str_replace(' .', ' ', str_replace('. ', ' ', $data));

    223. 

  223.     }

    224. 

  224.     else if ($isearch_config['allow_dots'] == 3)

    225. 

  225.     {

    226. 

  226.         /* Remove All */

    227. 

  227.         $data = str_replace('.', '', $data);

    228. 

  228.     }

    229. 

  229. 

    230. 

  230.     if ($isearch_config['allow_commas'] == 0)

    231. 

  231.     {

    232. 

  232.         /* Replace with spaces */

    233. 

  233.         $data = str_replace(',', ' ', $data);

    234. 

  234.     }

    235. 

  235.     else if ($isearch_config['allow_commas'] == 1)

    236. 

  236.     {

    237. 

  237.         /* Allow within words */

    238. 

  238.         $data = str_replace(' ,', ' ', str_replace(', ', ' ', $data));

    239. 

  239.     }

    240. 

  240.     else if ($isearch_config['allow_commas'] == 3)

    241. 

  241.     {

    242. 

  242.         /* Remove All */

    243. 

  243.         $data = str_replace(',', '', $data);

    244. 

  244.     }

    245. 

  245. 

    246. 

  246.     if ($isearch_config['allow_underscores'] == 0)

    247. 

  247.     {

    248. 

  248.         /* Replace with spaces */

    249. 

  249.         $data = str_replace('_', ' ', $data);

    250. 

  250.     }

    251. 

  251.     else if ($isearch_config['allow_underscores'] == 1)

    252. 

  252.     {

    253. 

  253.         /* Allow within words */

    254. 

  254.         $data = str_replace(' _', ' ', str_replace('_ ', ' ', $data));

    255. 

  255.     }

    256. 

  256.     else if ($isearch_config['allow_underscores'] == 3)

    257. 

  257.     {

    258. 

  258.         /* Remove All */

    259. 

  259.         $data = str_replace('_', '', $data);

    260. 

  260.     }

    261. 

  261.     

    262. 

  262.     if ($isearch_config['char_set_8_bit'])

    263. 

  263.     {

    264. 

  264.         /* Strip out all characters except whitespace numeric and alpha */

    265. 

  265.         $data = preg_replace('/([^-@0-9a-z:\\.,' . chr(0xbf) . '-' . chr(0xff) . chr(0x9a) . chr(0x9e) . '\s])/', '', $data);

    266. 

  266.     }

    267. 

  267.     else

    268. 

  268.     {

    269. 

  269.         /* Remove any single quotes and backslashes. */

    270. 

  270.         $data = ereg_replace('[\'\\]', '', $data);

    271. 

  271.     }

    272. 

  272. 

    273. 

  273.     /* Convert from source charset to charset used on results page */

    274. 

  274.     if ((strtolower($charset) != $isearch_config['char_set']) &&

    275. 

  275.         ($charset != '') &&

    276. 

  276.         ($isearch_config['char_set'] != ''))

    277. 

  277.     {

    278. 

  278.         if (function_exists('iconv'))

    279. 

  279.         {

    280. 

  280.             isearch_log("INFO: Converting $charset -> " . $isearch_config['char_set'], 5);

    281. 

  281.             $convertedData = iconv($charset, $isearch_config['char_set'], $data);

    282. 

  282.             if ($convertedData === False)

    283. 

  283.             {

    284. 

  284.                 isearch_log("WARNING: Unable to convert $charset -> " . $isearch_config['char_set'], 3);

    285. 

  285.             }

    286. 

  286.             else

    287. 

  287.             {

    288. 

  288.                 $data = $convertedData;

    289. 

  289.             }

    290. 

  290.         }

    291. 

  291.         else

    292. 

  292.         {

    293. 

  293.             isearch_log("WARNING: iconv not installed - unable to convert $charset -> " . $isearch_config['char_set'], 5);

    294. 

  294.         }

    295. 

  295.     }

    296. 

  296. 

    297. 

  297.     /* Compact all white space into a single space character */

    298. 

  298.     $data = preg_replace("/\\s+/", ' ', $data);

    299. 

  299. 

    300. 

  300.     /* Strip white space from beginning and end of the string */

    301. 

  301.     $data = trim($data);

    302. 

  302. 

    303. 

  303.     return $data;

    304. 

  304. }

    305. 

  305. 

    306. 

  306. 

    307. 

  307. function isearch_fread($handle, $length = 2147483647)

    308. 

  308. {

    309. 

  309.     $bytesToRead = $length;

    310. 

  310. 

    311. 

  311.     $contents = '';

    312. 

  312.     while((!feof($handle)) && ($bytesToRead > 0))

    313. 

  313.     {

    314. 

  314.         $data = fread($handle, ($bytesToRead > 16384) ? 16384 : $bytesToRead);

    315. 

  315.         $bytesToRead -= strlen($data);

    316. 

  316.         $contents .= $data;

    317. 

  317.     }

    318. 

  318.     if (!feof($handle))

    319. 

  319.     {

    320. 

  320.         isearch_log('WARNING:    File reading was truncated at '.($length/1024).' kbytes', 3);

    321. 

  321.     }

    322. 

  322.     return $contents;

    323. 

  323. }

    324. 

  324. 

    325. 

  325. 

    326. 

  326. function isearch_relativeToAbsoluteUrl($newUrl, $relativeToUrl)

    327. 

  327. {

    328. 

  328.     /* Convert to absolute reference */

    329. 

  329.     if (eregi('^([a-z]+):', $newUrl))

    330. 

  330.     {

    331. 

  331.         $absoluteUrl = $newUrl;

    332. 

  332.     }

    333. 

  333.     else

    334. 

  334.     {

    335. 

  335.         $relativeParts = @parse_url($relativeToUrl);

    336. 

  336.         if ((!isset($relativeParts['scheme'])) || (!isset($relativeParts['host'])))

    337. 

  337.         {

    338. 

  338.             /* Unable to parse relativeToUrl */

    339. 

  339.             isearch_log("WARNING:    Unable to parse relativeToUrl [$relativeToUrl]", 3);

    340. 

  340.             return;

    341. 

  341.         }

    342. 

  342.         unset($relativeParts['query']);

    343. 

  343.         unset($relativeParts['fragment']);

    344. 

  344. 

    345. 

  345.         if (ereg('^/', $newUrl))

    346. 

  346.         {

    347. 

  347.             /* New URL begins with a slash. It is within the site */

    348. 

  348. 

    349. 

  349.             unset($relativeParts['path']);

    350. 

  350.             $siteUrl = glue_url($relativeParts);

    351. 

  351. 

    352. 

  352.             $absoluteUrl = $siteUrl . $newUrl;

    353. 

  353.         }

    354. 

  354.         else

    355. 

  355.         {

    356. 

  356.             /* A relative reference (must be within this site) */

    357. 

  357. 

    358. 

  358.             /* Remove filename following the last slash */

    359. 

  359.             if (isset($relativeParts['path']))

    360. 

  360.             {

    361. 

  361.                 $path = ereg_replace('/[^/]*\.[^/]*$', '/', $relativeParts['path']);

    362. 

  362.                 if (ereg('/$', $path))

    363. 

  363.                 {

    364. 

  364.                     $path .= $newUrl;

    365. 

  365.                 }

    366. 

  366.                 else

    367. 

  367.                 {

    368. 

  368.                     $path .= '/' . $newUrl;

    369. 

  369.                 }

    370. 

  370.             }

    371. 

  371.             else

    372. 

  372.             {

    373. 

  373.                 $path = '/' . $newUrl;

    374. 

  374.             }

    375. 

  375.             $path = ereg_replace('/\.$', '', $path);           /* Remove ending "/." */

    376. 

  376.             $path = ereg_replace('/(\./)+', '/', $path);       /* Remove any "." references */

    377. 

  377.             $path = ereg_replace('/+/', '/', $path);           /* Remove excess slashes */

    378. 

  378. 

    379. 

  379.             /* Resolve any ".." references */

    380. 

  380.             $temp = explode('/', $path);

    381. 

  381.             for ($i = 1; $i < count($temp); $i++)

    382. 

  382.             {

    383. 

  383.                 if (($temp[$i] == "..") && ($i > 1))

    384. 

  384.                 {

    385. 

  385.                     for ($j = $i + 1; $j < count($temp); $j ++)

    386. 

  386.                     {

    387. 

  387.                         $temp[$j-2] = $temp[$j];

    388. 

  388.                     }

    389. 

  389.                     unset($temp[count($temp)-1]);

    390. 

  390.                     unset($temp[count($temp)-1]);

    391. 

  391.                     $i = $i - 2;

    392. 

  392.                 }

    393. 

  393.             }

    394. 

  394.             $relativeParts['path'] = implode('/', $temp);

    395. 

  395.             $absoluteUrl = glue_url($relativeParts);

    396. 

  396.         }

    397. 

  397.     }

    398. 

  398. 

    399. 

  399.     return $absoluteUrl;

    400. 

  400. }

    401. 

  401. 

    402. 

  402. 

    403. 

  403. function isearch_readFile($url, $depth=0)

    404. 

  404. {

    405. 

  405.     global $isearch_config;

    406. 

  406.     global $isearch_version;

    407. 

  407.     global $isearch_header;

    408. 

  408.     global $isearch_base;

    409. 

  409.     global $isearch_url_fopen_detected, $isearch_curl_detected, $isearch_sockets_detected;

    410. 

  410. 

    411. 

  411.     $isearch_header = array();

    412. 

  412. 

    413. 

  413.     isearch_log("TRACE: isearch_readFile($url, $depth)", 10);

    414. 

  414. 

    415. 

  415.     if ($depth >= 10)

    416. 

  416.     {

    417. 

  417.         /* Inline frame depth of 10 */

    418. 

  418.         isearch_log("WARNING: Inline frame depth limit $depth exceeded", 3);

    419. 

  419.         return '';

    420. 

  420.     }

    421. 

  421. 

    422. 

  422.     if ($isearch_config['url_search'] != '')

    423. 

  423.     {

    424. 

  424.         $url = ereg_replace($isearch_config['url_search'], $isearch_config['url_replace'], $url);

    425. 

  425.         isearch_log("INFO: Using replaced URL $url", 5);

    426. 

  426.     }

    427. 

  427. 

    428. 

  428. 

    429. 

  429.     if ($isearch_config['reading_mechanism'] == 0)

    430. 

  430.     {

    431. 

  431.         /* Autodetect */

    432. 

  432.         if (($isearch_url_fopen_detected) && (!$isearch_config['proxy_enable']))

    433. 

  433.         {

    434. 

  434.             $reading_mechanism = 1;   /* fopen */

    435. 

  435.         }

    436. 

  436.         else if (($isearch_sockets_detected) && (eregi('^http://', $url)))

    437. 

  437.         {

    438. 

  438.             $reading_mechanism = 2;   /* sockets */

    439. 

  439.         }

    440. 

  440.         else if ($isearch_curl_detected)

    441. 

  441.         {

    442. 

  442.             $reading_mechanism = 3;   /* curl */

    443. 

  443.         }

    444. 

  444.         else

    445. 

  445.         {

    446. 

  446.             isearch_log('ERROR: Unable to detect a suitable reading mechanism.', 1);

    447. 

  447.             return '';

    448. 

  448.         }

    449. 

  449.     }

    450. 

  450.     else

    451. 

  451.     {

    452. 

  452.         $reading_mechanism = $isearch_config['reading_mechanism'];

    453. 

  453.     }

    454. 

  454. 

    455. 

  455.     $base = $url;

    456. 

  456. 

    457. 

  457.     if ($reading_mechanism == 1)

    458. 

  458.     {

    459. 

  459.         /* Use fopen/fread */

    460. 

  460.         isearch_log("INFO: Reading $url using fopen/fread", 5);

    461. 

  461.         $docData = '';

    462. 

  462.         @ini_set('user_agent', "iSearch/$isearch_version");

    463. 

  463.         if ($isearch_config['basic_authorization'] != '')

    464. 

  464.         {

    465. 

  465.             $url = str_replace('//', '//'.$isearch_config['basic_authorization'].'@', $url);

    466. 

  466.         }

    467. 

  467.         $fp = @fopen($url, 'r');

    468. 

  468.         if ($fp)

    469. 

  469.         {

    470. 

  470.             if (function_exists('stream_get_meta_data'))

    471. 

  471.             {

    472. 

  472.                 /* Prior to PHP 4.3.0 use $http_response_header instead of stream_get_meta_data() */

    473. 

  473.                 $meta_data = stream_get_meta_data($fp);

    474. 

  474.                 $header_data = $meta_data['wrapper_data'];

    475. 

  475.             }

    476. 

  476.             else

    477. 

  477.             {

    478. 

  478.                 $header_data = $http_response_header;

    479. 

  479.             }

    480. 

  480. 

    481. 

  481.             $header = array();

    482. 

  482.             foreach($header_data as $headerLine)

    483. 

  483.             {

    484. 

  484.                 $data = explode(': ', $headerLine, 2);

    485. 

  485.                 if (count($data) == 2)

    486. 

  486.                 {

    487. 

  487.                     $header[strtolower($data[0])] = $data[1];

    488. 

  488.                 }

    489. 

  489.             }

    490. 

  490. 

    491. 

  491.             if (isset($header['content-location']))

    492. 

  492.             {

    493. 

  493.                 isearch_log("INFO: Content-Location: ".$header['content-location'], 9);

    494. 

  494.                 $base = isearch_relativeToAbsoluteUrl($header['content-location'], $base);

    495. 

  495.             }

    496. 

  496.             else if (isset($header['location']))

    497. 

  497.             {

    498. 

  498.                 isearch_log("INFO: Location: ".$header['location'], 9);

    499. 

  499.                 $base = isearch_relativeToAbsoluteUrl($header['location'], $base);

    500. 

  500.             }

    501. 

  501. 

    502. 

  502.             $docData = isearch_fread($fp, $isearch_config['max_file_size']);

    503. 

  503.             fclose($fp);

    504. 

  504.         }

    505. 

  505.         else

    506. 

  506.         {

    507. 

  507.             isearch_log("WARNING: Unable to fopen URL [$url]", ereg('/robots\.txt$', $url) ? 9 : 3);

    508. 

  508.             return '';

    509. 

  509.         }

    510. 

  510.     }

    511. 

  511.     else

    512. 

  512.     {

    513. 

  513.         $recurse = 10;

    514. 

  514.         while (1)

    515. 

  515.         {

    516. 

  516.             /* Check URL and determine whether this is a file or directory */

    517. 

  517.             $urlParts = @parse_url($url);

    518. 

  518.             if ((!isset($urlParts['scheme'])) || (!isset($urlParts['host'])))

    519. 

  519.             {

    520. 

  520.                 isearch_log("WARNING: Unable to parse URL [$url]", 3);

    521. 

  521.                 return '';

    522. 

  522.             }

    523. 

  523. 

    524. 

  524.             if (!eregi('^(https?|ftps?)$', $urlParts['scheme']))

    525. 

  525.             {

    526. 

  526.                 isearch_log("WARNING: Unsupported URL scheme " . $urlParts['scheme'] . " [$url]", 4);

    527. 

  527.                 return '';

    528. 

  528.             }

    529. 

  529. 

    530. 

  530.             if ($reading_mechanism == 2)

    531. 

  531.             {

    532. 

  532.                 isearch_log("INFO: Reading $url using sockets", 5);

    533. 

  533. 

    534. 

  534.                 if ($urlParts['scheme'] == 'http')

    535. 

  535.                 {

    536. 

  536.                     $secure = False;

    537. 

  537.                 }

    538. 

  538.                 else if ($urlParts['scheme'] == 'https')

    539. 

  539.                 {

    540. 

  540.                     $secure = True;

    541. 

  541.                 }

    542. 

  542.                 else

    543. 

  543.                 {

    544. 

  544.                     isearch_log("WARNING: URL scheme " . $urlParts['scheme'] . " not supported by sockets. Use CURL library. [$url]", 3);

    545. 

  545.                     return '';

    546. 

  546.                 }

    547. 

  547. 

    548. 

  548.                 if ($isearch_config['proxy_enable'])

    549. 

  549.                 {

    550. 

  550.                     $host = $isearch_config['proxy_host'];

    551. 

  551.                     $port = $isearch_config['proxy_port'];

    552. 

  552.                 }

    553. 

  553.                 else

    554. 

  554.                 {

    555. 

  555.                     $host = $urlParts['host'];

    556. 

  556.                     if (isset($urlParts['port']))

    557. 

  557.                     {

    558. 

  558.                         $port =  $urlParts['port'];

    559. 

  559.                     }

    560. 

  560.                     else

    561. 

  561.                     {

    562. 

  562.                         $port = $secure ? 443 : 80;

    563. 

  563.                     }

    564. 

  564.                 }

    565. 

  565. 

    566. 

  566.                 $sock = fsockopen(($secure ? 'ssl://' : '').$host, $port, $errno, $errstr);

    567. 

  567.                 if (!$sock)

    568. 

  568.                 {

    569. 

  569.                     isearch_log("ERROR: Unable to open socket to " . $host . " " . $port . " - $errno : $errstr", 1);

    570. 

  570.                     return '';

    571. 

  571.                 }

    572. 

  572. 

    573. 

  573.                 $request = "GET $url HTTP/1.0\r\n";

    574. 

  574.                 $request .= "Host: $host\r\n";

    575. 

  575.                 if (($isearch_config['proxy_enable']) && ($isearch_config['proxy_user'] != ''))

    576. 

  576.                 {

    577. 

  577.                     $request .= "Proxy-Authorization: Basic " . base64_encode ($isearch_config['proxy_user'].':'.$isearch_config['proxy_pass']) . "\r\n";

    578. 

  578.                 }

    579. 

  579. 

    580. 

  580.                 $request .= "User-Agent: iSearch/$isearch_version\r\n";

    581. 

  581.                 if ($isearch_config['basic_authorization'] != '')

    582. 

  582.                 {

    583. 

  583.                     $request .= "Authorization: Basic " . base64_encode($isearch_config['basic_authorization']) . "\r\n";

    584. 

  584.                 }

    585. 

  585.                 $request .= "Connection: Close\r\n\r\n";

    586. 

  586. 

    587. 

  587.                 fputs($sock, $request);

    588. 

  588. 

    589. 

  589.                 $allData = isearch_fread($sock, $isearch_config['max_file_size']);

    590. 

  590.                 fclose($sock);

    591. 

  591.             }

    592. 

  592.             else

    593. 

  593.             {

    594. 

  594.                 /* Use the CURL library */

    595. 

  595.                 isearch_log("INFO: Reading $url using CURL", 5);

    596. 

  596.                 $ch = curl_init($url);

    597. 

  597. 

    598. 

  598. //                curl_setopt($ch, CURLOPT_VERBOSE, 1);

    599. 

  599.                 curl_setopt($ch, CURLOPT_USERAGENT, "iSearch/$isearch_version");

    600. 

  600.                 curl_setopt($ch, CURLOPT_HEADER, TRUE);

    601. 

  601.                 curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);

    602. 

  602.                 curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);

    603. 

  603.                 curl_setopt($ch, CURLOPT_HEADER, TRUE);

    604. 

  604.                 curl_setopt($ch, CURLOPT_TIMEOUT, 30);

    605. 

  605. 

    606. 

  606.                 curl_setopt($ch, CURLOPT_COOKIEJAR,  "my_cookies.txt");     // Initiates cookie file if needed

    607. 

  607.                 curl_setopt($ch, CURLOPT_COOKIEFILE, "my_cookies.txt");     // Uses cookies from previous session if exist

    608. 

  608. 

    609. 

  609.                 if ($isearch_config['proxy_enable'])

    610. 

  610.                 {

    611. 

  611.                     if ($isearch_config['proxy_user'] != '')

    612. 

  612.                     {

    613. 

  613.                         curl_setopt($ch, CURLOPT_PROXYUSERPWD, $isearch_config['proxy_user'].':'.$isearch_config['proxy_pass']);

    614. 

  614.                     }

    615. 

  615.                     curl_setopt($ch, CURLOPT_PROXY, $isearch_config['proxy_host']);

    616. 

  616.                     curl_setopt($ch, CURLOPT_PROXYPORT, $isearch_config['proxy_port']);

    617. 

  617.                 }

    618. 

  618.                 if ($isearch_config['basic_authorization'] != '')

    619. 

  619.                 {

    620. 

  620.                     curl_setopt($ch, CURLOPT_USERPWD, $isearch_config['basic_authorization']);

    621. 

  621.                 }

    622. 

  622. 

    623. 

  623.                 $allData = curl_exec($ch);

    624. 

  624. 

    625. 

  625.                 curl_close($ch);

    626. 

  626.             }

    627. 

  627. 

    628. 

  628.             @list($headerData, $docData) = explode("\r\n\r\n", $allData, 2);

    629. 

  629.             $headerLines = explode("\r\n", $headerData);

    630. 

  630.             $status = $headerLines[0];

    631. 

  631.             isearch_log("INFO: Status Line $status", 8);

    632. 

  632. 

    633. 

  633.             $header = array();

    634. 

  634.             for ($i = count($headerLines)-1; $i > 0; $i--)

    635. 

  635.             {

    636. 

  636.                 $data = explode(': ', $headerLines[$i], 2);

    637. 

  637.                 if (count($data) == 2)

    638. 

  638.                 {

    639. 

  639.                     $header[strtolower($data[0])] = $data[1];

    640. 

  640.                 }

    641. 

  641.             }

    642. 

  642. 

    643. 

  643.             /* Check status code:

    644. 

  644.              * "200"   ; OK

    645. 

  645.              * "201"   ; Created

    646. 

  646.              * "202"   ; Accepted

    647. 

  647.              * "204"   ; No Content

    648. 

  648.              * "301"   ; Moved Permanently

    649. 

  649.              * "302"   ; Moved Temporarily

    650. 

  650.              * "304"   ; Not Modified

    651. 

  651.              * "400"   ; Bad Request

    652. 

  652.              * "401"   ; Unauthorized

    653. 

  653.              * "403"   ; Forbidden

    654. 

  654.              * "404"   ; Not Found

    655. 

  655.              * "500"   ; Internal Server Error

    656. 

  656.              * "501"   ; Not Implemented

    657. 

  657.              * "502"   ; Bad Gateway

    658. 

  658.              * "503"   ; Service Unavailable

    659. 

  659.              */

    660. 

  660.             $statusParts = explode(' ', $status, 3);

    661. 

  661.             if (count($statusParts) < 2)

    662. 

  662.             {

    663. 

  663.                 isearch_log('ERROR: Unable to read status code', 1);

    664. 

  664.                 return '';

    665. 

  665.             }

    666. 

  666. 

    667. 

  667.             $statusCode = $statusParts[1];

    668. 

  668. 

    669. 

  669.             if (($statusCode == '301') || ($statusCode == '302'))

    670. 

  670.             {

    671. 

  671.                 /* Redirection. Get new location */

    672. 

  672.                 if ($recurse <= 1)

    673. 

  673.                 {

    674. 

  674.                     /* Recursion limit reached */

    675. 

  675.                     isearch_log('ERROR: URL recursion limit 10 exceeded', 1);

    676. 

  676.                     return '';

    677. 

  677.                 }

    678. 

  678. 

    679. 

  679.                 $url = $header['location'];

    680. 

  680.                 $recurse = $recurse - 1;

    681. 

  681.             }

    682. 

  682.             else if ($statusCode >= 300)

    683. 

  683.             {

    684. 

  684.                 isearch_log("WARNING: HTTP Error : $status [$url]", ereg('/robots\.txt$', $url) ? 9 : 3);

    685. 

  685.                 return '';

    686. 

  686.             }

    687. 

  687.             else

    688. 

  688.             {

    689. 

  689.                 /* We have read the file */

    690. 

  690.                 break;

    691. 

  691.             }

    692. 

  692.         }

    693. 

  693. 

    694. 

  694.         if (isset($header['content-location']))

    695. 

  695.         {

    696. 

  696.             isearch_log("INFO: Content-Location: ".$header['content-location'], 9);

    697. 

  697.             $base = $header['content-location'];

    698. 

  698.         }

    699. 

  699.         else

    700. 

  700.         {

    701. 

  701.             isearch_log("INFO: Url: ".$url, 9);

    702. 

  702.             $base = $url;

    703. 

  703.         }

    704. 

  704.     }

    705. 

  705. 

    706. 

  706.     if (preg_match("#<BASE\\s+[^>]*?HREF\\s*=\\s*['\"]?([^>]+?)['\"]?[\\s>]#i", $docData, $matches) == 1)

    707. 

  707.     {

    708. 

  708.         /* Found a "<BASE HREF=" tag in the document head */

    709. 

  709.         isearch_log("INFO: BASE: ".$matches[1], 9);

    710. 

  710.         $base = $matches[1];

    711. 

  711.     }

    712. 

  712. 

    713. 

  713.     /* Search for inline frames and replace them with frame contents */

    714. 

  714.     $regexp = "#<IFRAME[^>]*?\\sSRC\\s*=\\s*['\"]?(.*?)[\\s'\"][^>]*>#i";

    715. 

  715.     $matchCount = preg_match_all($regexp, $docData, $matches);

    716. 

  716.     for ($i = 0; $i < $matchCount; $i++)

    717. 

  717.     {

    718. 

  718.         $frameUrl = isearch_relativeToAbsoluteUrl($matches[1][$i], $base);

    719. 

  719.         isearch_log("INFO: Reading inline frame : $frameUrl", 5);

    720. 

  720.         $frameData = isearch_readFile($frameUrl, $depth+1);

    721. 

  721.         $docData = preg_replace($regexp, $frameData, $docData, 1);

    722. 

  722.     }

    723. 

  723. 

    724. 

  724.     if ($isearch_config['javascript_link_search'] == 2)

    725. 

  725.     {

    726. 

  726.         /* Search for external JavaScript files and replace them with file contents */

    727. 

  727.         $regexp = "#<SCRIPT[^>]*?\\sSRC\\s*=\\s*['\"](.*?)['\"][^>]*>#i";

    728. 

  728.         $matchCount = preg_match_all($regexp, $docData, $matches);

    729. 

  729.         for ($i = 0; $i < $matchCount; $i++)

    730. 

  730.         {

    731. 

  731.             $jsUrl = isearch_relativeToAbsoluteUrl($matches[1][$i], $base);

    732. 

  732.             isearch_log("INFO: Reading javascript : $jsUrl", 5);

    733. 

  733.             $jsData = isearch_readFile($jsUrl, $depth+1);

    734. 

  734.             $docData = preg_replace($regexp, "<SCRIPT>\n<!--\n".$jsData."\n-->\n</SCRIPT>\n", $docData, 1);

    735. 

  735.         }

    736. 

  736.     }

    737. 

  737. 

    738. 

  738.     $isearch_header = $header;

    739. 

  739.     $isearch_base = $base;

    740. 

  740. 

    741. 

  741.     return $docData;

    742. 

  742. }

    743. 

  743. 

    744. 

  744. 

    745. 

  745. function isearch_execConvert($data, $type, $url)

    746. 

  746. {

    747. 

  747.     global $isearch_config;

    748. 

  748. 

    749. 

  749.     isearch_log("INFO: Exec conversion $type", 7);

    750. 

  750. 

    751. 

  751.     // Create a temporary filename to use for the conversion

    752. 

  752.     $tmpfname = tempnam($isearch_config['tmpdir'], "iSearch");

    753. 

  753. 

    754. 

  754.     // Write data to the temp file

    755. 

  755.     $fh = fopen($tmpfname, 'wb');

    756. 

  756.     if (!$fh)

    757. 

  757.     {

    758. 

  758.         isearch_log("ERROR: Unable to open tmp file $tmpfname", 1);

    759. 

  759.         return '';

    760. 

  760.     }

    761. 

  761.     fwrite($fh, $data);

    762. 

  762.     fclose($fh);

    763. 

  763. 

    764. 

  764.     // Execute the command

    765. 

  765.     $cmd = $isearch_config[$type . '_exec'];

    766. 

  766.     if ($cmd == '')

    767. 

  767.     {

    768. 

  768.         isearch_log("ERROR: Configuration error - executable for $type not set", 1);

    769. 

  769.         return '';

    770. 

  770.     }

    771. 

  771. 

    772. 

  772.     $pdf = False;

    773. 

  773.     if ($type == 'pdf')

    774. 

  774.     {

    775. 

  775.         $pdf = True;

    776. 

  776.         $cmd .= ' -htmlmeta '.escapeshellarg($tmpfname).' -';

    777. 

  777.     }

    778. 

  778.     else

    779. 

  779.     {

    780. 

  780.         $cmd .= ' '.escapeshellarg($tmpfname);

    781. 

  781.     }

    782. 

  782. 

    783. 

  783.     exec($cmd, $output, $retval);

    784. 

  784. 

    785. 

  785.     // Delete the temporary file

    786. 

  786.     unlink($tmpfname);

    787. 

  787. 

    788. 

  788.     // Read the stderr and stdout files

    789. 

  789.     if (is_file("$tmpfname.err"))

    790. 

  790.     {

    791. 

  791.         $err = trim(@implode(" ", file("$tmpfname.err")));

    792. 

  792.         unlink("$tmpfname.err");

    793. 

  793.     }

    794. 

  794.     else

    795. 

  795.     {

    796. 

  796.         $err = '';

    797. 

  797.     }

    798. 

  798.     

    799. 

  799.     if (is_file("$tmpfname.txt"))

    800. 

  800.     {

    801. 

  801.         $text = @implode("\n", file("$tmpfname.txt"));

    802. 

  802.         unlink("$tmpfname.txt");

    803. 

  803.     }

    804. 

  804.     else

    805. 

  805.     {

    806. 

  806.         $text = implode("\n", $output);

    807. 

  807.     }

    808. 

  808. 

    809. 

  809.     if ($retval == 0)

    810. 

  810.     {

    811. 

  811.         isearch_log("INFO: Executed command $cmd $tmpfname, Return Code $retval", 5);

    812. 

  812.     }

    813. 

  813.     else

    814. 

  814.     {

    815. 

  815.         isearch_log("ERROR: Executed command $cmd $tmpfname, Return Code $retval", 1);

    816. 

  816.         return '';

    817. 

  817.     }

    818. 

  818. 

    819. 

  819.     if ($err != '')

    820. 

  820.     {

    821. 

  821.         isearch_log("ERROR: Executed command $cmd $tmpfname, Error Msg: $err", 1);

    822. 

  822.         return '';

    823. 

  823.     }

    824. 

  824. 

    825. 

  825.     if ($text == '')

    826. 

  826.     {

    827. 

  827.         isearch_log("ERROR: Unable to read converted file", 2);

    828. 

  828.         return '';

    829. 

  829.     }

    830. 

  830. 

    831. 

  831.     if ($type != 'pdf')

    832. 

  832.     {

    833. 

  833.         // Wrap text in HTML

    834. 

  834.         $fname = ereg_replace('^.*[/\\\\]', '', $url);

    835. 

  835.         $text = "<HTML><HEAD><TITLE>$fname</TITLE></HEAD><BODY><PRE>$text</PRE></BODY></HTML>";

    836. 

  836.     }

    837. 

  837.     

    838. 

  838.     return $text;

    839. 

  839. }

    840. 

  840. 

    841. 

  841. 

    842. 

  842. function isearch_onlineConvert($data, $type, $url)

    843. 

  843. {

    844. 

  844.     global $isearch_config;

    845. 

  845.     global $isearch_version;

    846. 

  846. 

    847. 

  847.     $host='convert.iSearchTheNet.com';

    848. 

  848.     $port=80;

    849. 

  849.     $path='/';

    850. 

  850.     $query="?type=$type&gzip=0";

    851. 

  851. 

    852. 

  852.     isearch_log("INFO: Online conversion $type", 7);

    853. 

  853.     

    854. 

  854.     $sock = fsockopen($host, $port, $errno, $errstr);

    855. 

  855.     if (!$sock)

    856. 

  856.     {

    857. 

  857.         isearch_log("ERROR: Unable to open socket $host $port - $errno : $errstr", 1);

    858. 

  858.         return '';

    859. 

  859.     }

    860. 

  860. 

    861. 

  861.     $boundary = '---------------------------' . md5('boundary');

    862. 

  862. 

    863. 

  863.     $postValues  = "--$boundary\r\n";

    864. 

  864.     $postValues .= "Content-Disposition: form-data; name=\"file.1\"; filename=\"file.1\"\r\n";

    865. 

  865.     $postValues .= "Content-Type: application/$type\r\n";

    866. 

  866.     $postValues .= "\r\n";

    867. 

  867.     $postValues .= "$data\r\n";

    868. 

  868.     $postValues .= "--$boundary--\r\n\r\n";

    869. 

  869. 

    870. 

  870.     $request  = "POST http://$host$path$query HTTP/1.0\r\n";

    871. 

  871.     $request .= "User-Agent: iSearch/$isearch_version\r\n";

    872. 

  872.     $request .= "Host: $host\r\n";

    873. 

  873.     $request .= "Authorization: ISEARCH " . $isearch_config['online_id'] . "\r\n";

    874. 

  874.     $request .= "Content-Type: multipart/form-data; boundary=$boundary\r\n";

    875. 

  875.     $request .= "Content-Length: " . strlen( $postValues ) . "\r\n";

    876. 

  876. //    $request .= "Connection: Close\r\n";

    877. 

  877.     $request .= "\r\n";

    878. 

  878. 

    879. 

  879.     fputs($sock, $request.$postValues);

    880. 

  880. 

    881. 

  881.     /* Read status line */

    882. 

  882.     $status = fgets($sock, 1024);

    883. 

  883. 

    884. 

  884.     /* Read the header */

    885. 

  885.     while (!feof($sock))

    886. 

  886.     {

    887. 

  887.         $line = trim(fgets($sock, 1024));

    888. 

  888.         if ($line == '')

    889. 

  889.         {

    890. 

  890.             break;

    891. 

  891.         }

    892. 

  892.     }

    893. 

  893.     $convertedData = isearch_fread($sock);

    894. 

  894.     fclose($sock);

    895. 

  895. 

    896. 

  896.     $statusParts = explode(' ', $status, 3);

    897. 

  897.     if ($statusParts[1] != '200')

    898. 

  898.     {

    899. 

  899.         isearch_log("ERROR: Online conversion error: $status", 1);

    900. 

  900.         return $data;

    901. 

  901.     }

    902. 

  902. 

    903. 

  903.     if ((strlen($convertedData) > 10) && ($convertedData[0] == 0x1f) && ($convertedData[1] == 0x8b))

    904. 

  904.     {

    905. 

  905.         isearch_log("INFO: Running gzinflate on converted data", 6);

    906. 

  906.         $convertedData = gzinflate(substr($convertedData,10));

    907. 

  907.     }

    908. 

  908. 

    909. 

  909.     // Check the first word of the converted data

    910. 

  910.     $firstWord = '';

    911. 

  911.     sscanf($convertedData, ' %s ', $firstWord);

    912. 

  912.     if (!eregi('<HTML>', $firstWord))

    913. 

  913.     {

    914. 

  914.         // Wrap text in HTML

    915. 

  915.         $fname = ereg_replace('^.*[/\\\\]', '', $url);

    916. 

  916.         $convertedData = "<HTML><HEAD><TITLE>$fname</TITLE></HEAD><BODY><PRE>$convertedData</PRE></BODY></HTML>";

    917. 

  917.     }

    918. 

  918. 

    919. 

  919.     return $convertedData;

    920. 

  920. }

    921. 

  921. 

    922. 

  922. function isearch_utf8_chr($ch)

    923. 

  923. {

    924. 

  924.     if ($ch <= 0x7f)

    925. 

  925.     {

    926. 

  926.         return chr($ch);

    927. 

  927.     }

    928. 

  928. 

    929. 

  929.     if ($ch <= 0x7ff)

    930. 

  930.     {

    931. 

  931.         return chr(($ch >> 6) + 0xc0).chr(($ch & 0x3f) + 0x80);

    932. 

  932.     }

    933. 

  933. 

    934. 

  934.     if ($ch <= 0xffff)

    935. 

  935.     {

    936. 

  936.         return chr(($ch >> 12) + 0xe0).chr((($ch >> 6) & 0x3f) + 0x80).chr(($ch & 0x3f) + 0x80);

    937. 

  937.     }

    938. 

  938. 

    939. 

  939.     if ($ch <= 0x1fffff)

    940. 

  940.     {

    941. 

  941.         return chr(($ch >> 18) + 0xf0).chr((($ch >> 12) & 0x3f) + 0x80).chr((($ch >> 6) & 0x3f) + 0x80).chr(($num & 0x3f) + 0x80);

    942. 

  942.     }

    943. 

  943. 

    944. 

  944.     // Invalid UTF-8 code

    945. 

  945.     return '';

    946. 

  946. }

    947. 

  947. 

    948. 

  948. function isearch_html_entity_decode($string, $quote, $charset)

    949. 

  949. {

    950. 

  950.     $version = phpversion();

    951. 

  951.     if ($version{0} == '4')

    952. 

  952.     {

    953. 

  953.         if (strtolower($charset) == 'utf-8')

    954. 

  954.         {

    955. 

  955.             // PHP4 html_entity_decode does not support multi-byte charsets

    956. 

  956.             static $utf8_trans_tbl;

    957. 

  957. 

    958. 

  958.             $string = preg_replace('/&#x([0-9a-f]+);/ei', 'isearch_utf8_chr(hexdec("\\1"))', $string);

    959. 

  959.             $string = preg_replace('/&#([0-9]+);/e', 'isearch_utf8_chr(\\1)', $string);

    960. 

  960.             

    961. 

  961.             if (!isset($utf8_trans_tbl))

    962. 

  962.             {

    963. 

  963.                 $utf8_trans_tbl = array();

    964. 

  964. 

    965. 

  965.                 foreach (get_html_translation_table(HTML_ENTITIES) as $val=>$key)

    966. 

  966.                 {

    967. 

  967.                     $utf8_trans_tbl[$key] = utf8_encode($val);

    968. 

  968.                 }

    969. 

  969.             }

    970. 

  970. 

    971. 

  971.             return strtr($string, $utf8_trans_tbl);

    972. 

  972.         }

    973. 

  973.         

    974. 

  974.         $string = preg_replace('/&#x([0-9a-f]+);?/ei', 'chr(hexdec("\\1"))', $string);

    975. 

  975.         $string = preg_replace('/&#([0-9]+);?/e', 'chr("\\1")', $string);

    976. 

  976.     }

    977. 

  977. 

    978. 

  978.     if (!function_exists('html_entity_decode'))

    979. 

  979.     {

    980. 

  980.         // html_entity_decode was new in PHP 4.3.0

    981. 

  981. 

    982. 

  982.         global $isearch_htmlToAsciiTrans;

    983. 

  983.         if (!isset($isearch_htmlToAsciiTrans))

    984. 

  984.         {

    985. 

  985.             /* Translate from HTML to ASCII */

    986. 

  986.             $isearch_htmlToAsciiTrans = array_flip(get_html_translation_table(HTML_ENTITIES));

    987. 

  987.         }

    988. 

  988.         return strtr($string, $isearch_htmlToAsciiTrans);

    989. 

  989.     }

    990. 

  990. 

    991. 

  991.     return html_entity_decode($string, $quote, $charset);

    992. 

  992. }

    993. 

  993. 

    994. 

  994. 

    995. 

  995. /* Parse HTTP date format - one of the following:

    996. 

  996.  *      Sun, 06 Nov 1994 08:49:37 GMT  ; RFC 822, updated by RFC 1123

    997. 

  997.  *      Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036

    998. 

  998.  *      Sun Nov  6 08:49:37 1994       ; ANSI C's asctime() format

    999. 

  999.  */

    1000. 

  1000. function isearch_parseHttpDate($httpDate)

    1001. 

  1001. {

    1002. 

  1002.     static $months;

    1003. 

  1003. 

    1004. 

  1004.     if (!isset($months))

    1005. 

  1005.     {

    1006. 

  1006.         $months = array('jan'=>1, 'feb'=>2, 'mar'=>3, 'apr'=>4, 'may'=>5, 'jun'=>6, 'jul'=>7, 'aug'=>8, 'sep'=>9, 'oct'=>10, 'nov'=>11, 'dec'=>12);

    1007. 

  1007.     }

    1008. 

  1008.     

    1009. 

  1009.     $time = 0;

    1010. 

  1010.     if (preg_match("#^[a-z]+,? +([0-9]{1,2})[ -]+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[ -]+([0-9]{2,4}) +([0-9]{2}):([0-9]{2}):([0-9]{2})#i", $httpDate, $matchName) == 1)

    1011. 

  1011.     {

    1012. 

  1012.         $day = $matchName[1];

    1013. 

  1013.         $monthName = strtolower($matchName[2]);

    1014. 

  1014.         $year = $matchName[3];

    1015. 

  1015.         $hour = $matchName[4];

    1016. 

  1016.         $min = $matchName[5];

    1017. 

  1017.         $sec = $matchName[6];

    1018. 

  1018.     }

    1019. 

  1019.     else if (preg_match("#^[a-z]+,?[ -]+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) +([0-9]{2}):([0-9]{2}):([0-9]{2}) +([0-9]{2,4})#i", $httpDate, $matchName) == 1)

    1020. 

  1020.     {

    1021. 

  1021.         $day = $matchName[2];

    1022. 

  1022.         $monthName = strtolower($matchName[1]);

    1023. 

  1023.         $year = $matchName[6];

    1024. 

  1024.         $hour = $matchName[3];

    1025. 

  1025.         $min = $matchName[4];

    1026. 

  1026.         $sec = $matchName[5];

    1027. 

  1027.     }

    1028. 

  1028.     else

    1029. 

  1029.     {

    1030. 

  1030.         isearch_log('WARNING: Unknown date format : ' . $httpDate, 7);

    1031. 

  1031.     }

    1032. 

  1032. 

    1033. 

  1033.     if (isset($day))

    1034. 

  1034.     {

    1035. 

  1035.         if ($year < 70)

    1036. 

  1036.         {

    1037. 

  1037.             $year += 2000;

    1038. 

  1038.         }

    1039. 

  1039.         else if ($year < 100)

    1040. 

  1040.         {

    1041. 

  1041.             $year += 1900;

    1042. 

  1042.         }

    1043. 

  1043. 

    1044. 

  1044.         $time = gmmktime($hour, $min, $sec, $months[$monthName], $day, $year);

    1045. 

  1045.     }

    1046. 

  1046. 

    1047. 

  1047.     return $time;

    1048. 

  1048. }

    1049. 

  1049. 

    1050. 

  1050. 

    1051. 

  1051. /* Spider a single file. Returns true if there are more files to spider, else false */

    1052. 

  1052. function isearch_indexAFile($verbose = True)

    1053. 

  1053. {

    1054. 

  1054.     global $isearch_table_info, $isearch_table_urls, $isearch_table_urls_new, $isearch_table_words, $isearch_table_words_new;

    1055. 

  1055.     global $isearch_db;

    1056. 

  1056.     global $isearch_config;

    1057. 

  1057.     global $isearch_header;

    1058. 

  1058.     global $isearch_base;

    1059. 

  1059. 

    1060. 

  1060.     if (! $verbose)

    1061. 

  1061.     {

    1062. 

  1062.         /* Disable display of messages. */

    1063. 

  1063.         $isearch_config['log_echo_level'] = 0;

    1064. 

  1064.     }

    1065. 

  1065. 

    1066. 

  1066.     $resultUrls = mysql_query("SELECT * FROM $isearch_table_urls_new WHERE state='new' LIMIT 1", $isearch_db);

    1067. 

  1067.     if (!$resultUrls)

    1068. 

  1068.     {

    1069. 

  1069.         /* MySQL error. Sleep and try again */

    1070. 

  1070.         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1071. 

  1071.         sleep(5);

    1072. 

  1072.         return True;

    1073. 

  1073.     }

    1074. 

  1074. 

    1075. 

  1075.     if (mysql_num_rows($resultUrls) != 1)

    1076. 

  1076.     {

    1077. 

  1077.         isearch_log('INFO: Indexing completed.', 2);

    1078. 

  1078. 

    1079. 

  1079.         /* Indexing has completed */

    1080. 

  1080.         $now = time();

    1081. 

  1081.         if (!mysql_query("UPDATE $isearch_table_info SET last_update='$now' WHERE id='1'", $isearch_db))

    1082. 

  1082.         {

    1083. 

  1083.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1084. 

  1084.         }

    1085. 

  1085. 

    1086. 

  1086.         $result = mysql_query("SELECT url, state FROM $isearch_table_urls_new WHERE state!='ok'", $isearch_db);

    1087. 

  1087.         if (!$result)

    1088. 

  1088.         {

    1089. 

  1089.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1090. 

  1090.         }

    1091. 

  1091.         else if (mysql_num_rows($result) > 0)

    1092. 

  1092.         {

    1093. 

  1093.             isearch_log('INFO: Deleting the following URLs:', 4);

    1094. 

  1094.             while ($item = mysql_fetch_object($result))

    1095. 

  1095.             {

    1096. 

  1096.                 isearch_log('INFO:  ' . $item->url . ' (' . $item->state . ')', 4);

    1097. 

  1097.             }

    1098. 

  1098.         }

    1099. 

  1099. 

    1100. 

  1100.         /* Delete any unfound references */

    1101. 

  1101.         if (!mysql_query("DELETE FROM $isearch_table_urls_new WHERE state!='ok'", $isearch_db))

    1102. 

  1102.         {

    1103. 

  1103.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1104. 

  1104.         }

    1105. 

  1105. 

    1106. 

  1106.         /* Update referrer_id fields */

    1107. 

  1107.         $result = mysql_query("SELECT id, temp_referrer_id FROM $isearch_table_urls_new", $isearch_db);

    1108. 

  1108.         if (!$result)

    1109. 

  1109.         {

    1110. 

  1110.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1111. 

  1111.         }

    1112. 

  1112.         else if (mysql_num_rows($result) > 0)

    1113. 

  1113.         {

    1114. 

  1114.             mysql_query("LOCK TABLES $isearch_table_urls_new WRITE", $isearch_db);

    1115. 

  1115.             while ($item = mysql_fetch_object($result))

    1116. 

  1116.             {

    1117. 

  1117.                 if (!mysql_query("UPDATE $isearch_table_urls_new SET referrer_id='$item->temp_referrer_id' WHERE id='$item->id'", $isearch_db))

    1118. 

  1118.                 {

    1119. 

  1119.                     isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1120. 

  1120.                 }

    1121. 

  1121.             }

    1122. 

  1122.             mysql_query("UNLOCK TABLES", $isearch_db);

    1123. 

  1123.         }

    1124. 

  1124. 

    1125. 

  1125.         if ($isearch_config['test_mode'] == 0)

    1126. 

  1126.         {

    1127. 

  1127.             /* Swap the old and new tables */

    1128. 

  1128.             $backup_words = $isearch_table_words . '_tmp_backup';

    1129. 

  1129.             $backup_urls = $isearch_table_urls . '_tmp_backup';

    1130. 

  1130.             if (!mysql_query("RENAME TABLE $isearch_table_words TO $backup_words, " .

    1131. 

  1131.                                           "$isearch_table_words_new TO $isearch_table_words, " .

    1132. 

  1132.                                           "$backup_words TO $isearch_table_words_new, " .

    1133. 

  1133.                                           "$isearch_table_urls TO $backup_urls, " .

    1134. 

  1134.                                           "$isearch_table_urls_new TO $isearch_table_urls, " .

    1135. 

  1135.                                           "$backup_urls TO $isearch_table_urls_new", $isearch_db))

    1136. 

  1136.             {

    1137. 

  1137.                 isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1138. 

  1138.             }

    1139. 

  1139.         }

    1140. 

  1140. 

    1141. 

  1141.         /* Empty the new words table */

    1142. 

  1142.         if (!mysql_query("DELETE FROM $isearch_table_words_new", $isearch_db))

    1143. 

  1143.         {

    1144. 

  1144.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1145. 

  1145.         }

    1146. 

  1146. 

    1147. 

  1147.         /* Empty the new urls table */

    1148. 

  1148.         if (!mysql_query("DELETE FROM $isearch_table_urls_new", $isearch_db))

    1149. 

  1149.         {

    1150. 

  1150.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1151. 

  1151.         }

    1152. 

  1152. 

    1153. 

  1153.         /* Optimize the tables */

    1154. 

  1154.         if (!mysql_query("OPTIMIZE TABLE $isearch_table_urls, $isearch_table_words", $isearch_db))

    1155. 

  1155.         {

    1156. 

  1156.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1157. 

  1157.         }

    1158. 

  1158. 

    1159. 

  1159.         return False;

    1160. 

  1160.     }

    1161. 

  1161. 

    1162. 

  1162.     $itemUrl = mysql_fetch_object($resultUrls);

    1163. 

  1163. 

    1164. 

  1164.     $allData = isearch_readFile($itemUrl->url);

    1165. 

  1165. 

    1166. 

  1166.     $newState = 'error';

    1167. 

  1167. 

    1168. 

  1168.     if ($allData == '')

    1169. 

  1169.     {

    1170. 

  1170.         isearch_log("ERROR: Unable to open URL [$itemUrl->url]", 1);

    1171. 

  1171.         $newState = 'notfound';

    1172. 

  1172.     }

    1173. 

  1173.     else

    1174. 

  1174.     {

    1175. 

  1175.         $size = strlen($allData);

    1176. 

  1176.         $sig = md5($allData);

    1177. 

  1177. 

    1178. 

  1178.         /* Look for a duplicate page */

    1179. 

  1179.         $resultSig = mysql_query("SELECT * FROM $isearch_table_urls_new WHERE sig='$sig' AND size='$size' AND NOT url='" . isearch_escape_string($itemUrl->url) . "'", $isearch_db);

    1180. 

  1180.         if (!$resultSig)

    1181. 

  1181.         {

    1182. 

  1182.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1183. 

  1183.         }

    1184. 

  1184.         else if (mysql_num_rows($resultSig) > 0)

    1185. 

  1185.         {

    1186. 

  1186.             isearch_log("INFO: Duplicate URL - will not be processed [$itemUrl->url]", 4);

    1187. 

  1187.             $newState = 'duplicate';

    1188. 

  1188.         }

    1189. 

  1189.         else

    1190. 

  1190.         {

    1191. 

  1191.             /* No duplicates found. */

    1192. 

  1192. 

    1193. 

  1193.             isearch_log("INFO: Processing URL [$itemUrl->url]", 2);

    1194. 

  1194. 

    1195. 

  1195. 

    1196. 

  1196.             if ((preg_match('%\.(doc|pdf)(\?.*|#.*)?$%i', $itemUrl->url, $matches)) ||

    1197. 

  1197.                 (preg_match('/^%(PDF)/i', $allData, $matches)))

    1198. 

  1198.             {

    1199. 

  1199.                 if (strtolower($matches[1]) == 'doc')

    1200. 

  1200.                 {

    1201. 

  1201.                     switch ($isearch_config['msword_support'])

    1202. 

  1202.                     {

    1203. 

  1203.                         case 0:

    1204. 

  1204.                             /* Do nothing */

    1205. 

  1205.                             isearch_log("WARNING: Word support disabled", 3);

    1206. 

  1206.                             break;

    1207. 

  1207.                         case 1:

    1208. 

  1208.                             $allData = isearch_execConvert($allData, 'msword', $itemUrl->url);

    1209. 

  1209.                             break;

    1210. 

  1210.                         case 2:

    1211. 

  1211.                             $allData = isearch_onlineConvert($allData, 'msword', $itemUrl->url);

    1212. 

  1212.                             break;

    1213. 

  1213.                         default:

    1214. 

  1214.                             isearch_log("ERROR: Illegal Word document support setting", 1);

    1215. 

  1215.                             break;

    1216. 

  1216.                     }

    1217. 

  1217.                 }

    1218. 

  1218.                 else

    1219. 

  1219.                 {

    1220. 

  1220.                     switch ($isearch_config['pdf_support'])

    1221. 

  1221.                     {

    1222. 

  1222.                         case 0:

    1223. 

  1223.                             /* Do nothing */

    1224. 

  1224.                             isearch_log("WARNING: PDF support disabled", 3);

    1225. 

  1225.                             break;

    1226. 

  1226.                         case 1:

    1227. 

  1227.                             $allData = isearch_execConvert($allData, 'pdf', $itemUrl->url);

    1228. 

  1228.                             break;

    1229. 

  1229.                         case 2:

    1230. 

  1230.                             $allData = isearch_onlineConvert($allData, 'pdf', $itemUrl->url);

    1231. 

  1231.                             break;

    1232. 

  1232.                         default:

    1233. 

  1233.                             isearch_log("ERROR: Illegal PDF support setting", 1);

    1234. 

  1234.                             break;

    1235. 

  1235.                     }

    1236. 

  1236.                 }

    1237. 

  1237.             }

    1238. 

  1238. 

    1239. 

  1239. 

    1240. 

  1240.             if (($isearch_config['keep_cache']) && ($isearch_config['test_mode'] == 0))

    1241. 

  1241.             {

    1242. 

  1242.                 $cache = $allData;

    1243. 

  1243.             }

    1244. 

  1244.             else

    1245. 

  1245.             {

    1246. 

  1246.                 $cache = '';

    1247. 

  1247.             }

    1248. 

  1248. 

    1249. 

  1249.             /* Strip out all control characters and replace with spaces.

    1250. 

  1250.              * Compact all white space into a single space character.

    1251. 

  1251.              */

    1252. 

  1252.             $allData = preg_replace("/\\s+/", ' ', $allData);

    1253. 

  1253. 

    1254. 

  1254.             if (isset($isearch_header['content_type']) && ($isearch_header['content_type'] == 'text/plain'))

    1255. 

  1255.             {

    1256. 

  1256.                 $headData = '';

    1257. 

  1257.                 $bodyData = $allData;

    1258. 

  1258.             }

    1259. 

  1259.             else

    1260. 

  1260.             {

    1261. 

  1261.                 $tdata = spliti('</head[^>]*>', $allData, 2);

    1262. 

  1262.                 if (count($tdata) < 2)

    1263. 

  1263.                 {

    1264. 

  1264.                     $tdata = spliti('<body', $allData, 2);

    1265. 

  1265.                     if (count($tdata) < 2)

    1266. 

  1266.                     {

    1267. 

  1267.                         isearch_log('WARNING: <BODY> and </HEAD> tags not found', 4);

    1268. 

  1268.                         $headData = '';

    1269. 

  1269.                         $bodyData = $allData;

    1270. 

  1270.                     }

    1271. 

  1271.                     else

    1272. 

  1272.                     {

    1273. 

  1273.                         $headData = $tdata[0];

    1274. 

  1274.                         $bodyData = '<body' . $tdata[1];

    1275. 

  1275.                     }

    1276. 

  1276.                 }

    1277. 

  1277.                 else

    1278. 

  1278.                 {

    1279. 

  1279.                     $headData = $tdata[0];

    1280. 

  1280.                     $bodyData = $tdata[1];

    1281. 

  1281.                 }

    1282. 

  1282.             }

    1283. 

  1283.             

    1284. 

  1284.             /* Strip out HTML comments from head data */

    1285. 

  1285.             $headData = preg_replace('/<!--.+?-->/','',$headData);

    1286. 

  1286.             $headData = preg_replace("/\\s+/", ' ', $headData);

    1287. 

  1287. 

    1288. 

  1288.             /* Strip out all HTML tags except ones we are interested in */

    1289. 

  1289.             /* Includes workaround for PHP bug. See http://bugs.php.net/bug.php?id=21311 */

    1290. 

  1290.             $headData = strip_tags(eregi_replace("<\!DOCTYPE [^>]*>", '', $headData), '<meta><title><base>');

    1291. 

  1291. 

    1292. 

  1292.             $keyWords = '';

    1293. 

  1293.             $description = '';

    1294. 

  1294.             $title = '';

    1295. 

  1295.             $index = True;

    1296. 

  1296.             $follow = True;

    1297. 

  1297. 

    1298. 

  1298.             $matchCount = preg_match_all("#<META\\s+([^>]*?)\\s*>#i", $allData, $matches);

    1299. 

  1299.             for ($i = 0; $i < $matchCount; $i++)

    1300. 

  1300.             {

    1301. 

  1301.                 if (preg_match("#CONTENT\\s*=\\s*(['\"])(.*?)\\1#i", $matches[1][$i], $matchContent) == 1)

    1302. 

  1302.                 {

    1303. 

  1303.                     $metaContent = $matchContent[2];

    1304. 

  1304.                     if (preg_match("#NAME\\s*=\\s*['\"]?(.*?)['\"]?(\\s|$)#i", $matches[1][$i], $matchName) == 1)

    1305. 

  1305.                     {

    1306. 

  1306.                         // <META NAME="keywords" CONTENT="keyword list">

    1307. 

  1307.                         // <META NAME="description" CONTENT="description">

    1308. 

  1308.                         // <META NAME="robots" CONTENT="nofollow,noindex,noarchive">

    1309. 

  1309. 

    1310. 

  1310.                         $metaName = strtolower($matchName[1]);

    1311. 

  1311.                         if ($metaName == 'keywords')

    1312. 

  1312.                         {

    1313. 

  1313.                             $keyWords = ereg_replace(',', ' ', $metaContent);

    1314. 

  1314.                         }

    1315. 

  1315.                         else if ($metaName == 'description')

    1316. 

  1316.                         {

    1317. 

  1317.                             $description = $metaContent;

    1318. 

  1318.                         }

    1319. 

  1319.                         else if ($metaName == 'robots')

    1320. 

  1320.                         {

    1321. 

  1321.                             if (eregi('noindex', $metaContent))

    1322. 

  1322.                             {

    1323. 

  1323.                                 $index = False;

    1324. 

  1324.                             }

    1325. 

  1325.                             if (eregi('nofollow', $metaContent))

    1326. 

  1326.                             {

    1327. 

  1327.                                 $follow = False;

    1328. 

  1328.                             }

    1329. 

  1329.                             if (eregi('noarchive', $metaContent))

    1330. 

  1330.                             {

    1331. 

  1331.                                 $cache = '';

    1332. 

  1332.                             }

    1333. 

  1333.                         }

    1334. 

  1334.                     }

    1335. 

  1335.                     else if (preg_match("#HTTP-EQUIV\\s*=(['\"])(.*?)\\1#i", $matches[1][$i], $matchEquiv) == 1)

    1336. 

  1336.                     {

    1337. 

  1337.                         $isearch_header[strtolower($matchEquiv[2])] = $metaContent;

    1338. 

  1338.                     }

    1339. 

  1339.                 }

    1340. 

  1340.             }

    1341. 

  1341. 

    1342. 

  1342.             // Determine character set

    1343. 

  1343.             $charset = $isearch_config['char_set'];

    1344. 

  1344.             if ((isset($isearch_header['content-type'])) &&

    1345. 

  1345.                 (preg_match("#(^|\\s)\\s*CHARSET\\s*=\\s*['\"]?(.*?)['\"]?(\\s|$)#i", $isearch_header['content-type'], $matches) == 1))

    1346. 

  1346.             {

    1347. 

  1347.                 $charset = $matches[2];

    1348. 

  1348.             }

    1349. 

  1349. 

    1350. 

  1350.             $lastModified = 0;

    1351. 

  1351.             if (isset($isearch_header['last-modified']))

    1352. 

  1352.             {

    1353. 

  1353.                 $lastModified = isearch_parseHttpDate($isearch_header['last-modified']);

    1354. 

  1354.             }

    1355. 

  1355. 

    1356. 

  1356.             $changefreq = '';

    1357. 

  1357.             $priority = -1;

    1358. 

  1358. 

    1359. 

  1359.             if ($follow)

    1360. 

  1360.             {

    1361. 

  1361.                 $tdata1 = spliti('(<!-- ISEARCH_BEGIN_FOLLOW -->|</nofollow>)', ' ' . $bodyData);

    1362. 

  1362.                 if (count($tdata1) > 1)

    1363. 

  1363.                 {

    1364. 

  1364.                     /* At least 1 found. */

    1365. 

  1365. 

    1366. 

  1366.                     $followData = '';

    1367. 

  1367. 

    1368. 

  1368.                     /* Check for an END_INDEX before the first BEGIN_INDEX */

    1369. 

  1369.                     $tdata2 = spliti('(<!-- ISEARCH_END_FOLLOW -->|<nofollow>)', $tdata1[0]);

    1370. 

  1370.                     if (count($tdata2) > 1)

    1371. 

  1371.                     {

    1372. 

  1372.                         /* And END_INDEX was found. Add anything before it into the follow data */

    1373. 

  1373.                         $followData .= $tdata2[0];

    1374. 

  1374.                     }

    1375. 

  1375. 

    1376. 

  1376.                     for ($i = 1; $i < count($tdata1); $i++)

    1377. 

  1377.                     {

    1378. 

  1378.                         $tdata2 = spliti('(<!-- ISEARCH_END_FOLLOW -->|<nofollow>)', $tdata1[$i]);

    1379. 

  1379.                         $followData .= $tdata2[0];

    1380. 

  1380.                     }

    1381. 

  1381.                 }

    1382. 

  1382.                 else

    1383. 

  1383.                 {

    1384. 

  1384.                     $followData = $bodyData;

    1385. 

  1385.                 }

    1386. 

  1386. 

    1387. 

  1387.                 $urls = array();

    1388. 

  1388.                 if ($isearch_config['follow_meta_refresh'] && isset($isearch_header['refresh']) &&

    1389. 

  1389.                     (preg_match("#; *URL *= *(.*)$#i", $isearch_header['refresh'], $matches) == 1))

    1390. 

  1390.                 {

    1391. 

  1391.                     $urls[] = $matches[1];

    1392. 

  1392.                 }

    1393. 

  1393. 

    1394. 

  1394.                 /* Do agressive link searching */

    1395. 

  1395.                 if ($isearch_config['aggressive_link_search'])

    1396. 

  1396.                 {

    1397. 

  1397.                     $matchCount = preg_match_all("~(https?|ftps?)://[^'\"\\s>]*~i", $allData, $matches);

    1398. 

  1398.                     for ($i = 0; $i < $matchCount; $i++)

    1399. 

  1399.                     {

    1400. 

  1400.                         $urls[] = $matches[0][$i];

    1401. 

  1401.                     }

    1402. 

  1402. 

    1403. 

  1403.                     /* Find any links with no quotes */

    1404. 

  1404.                     $matchCount = preg_match_all("~<(A|AREA)\\s+([^>]*?\\s+)*?HREF\\s*=\\s*([^\\s>]+?)~i", $followData, $matches);

    1405. 

  1405.                     for ($i = 0; $i < $matchCount; $i++)

    1406. 

  1406.                     {

    1407. 

  1407.                         $urls[] = $matches[3][$i];

    1408. 

  1408.                     }

    1409. 

  1409.                 }

    1410. 

  1410. 

    1411. 

  1411.                 /* Do JavaScript link searching */

    1412. 

  1412.                 if ($isearch_config['javascript_link_search'])

    1413. 

  1413.                 {

    1414. 

  1414.                     /* Search for window.open() calls */

    1415. 

  1415.                     $matchCount = preg_match_all("~window.open\\s*\\(\\s*'(.+?)',~i", $allData, $matches);

    1416. 

  1416.                     for ($i = 0; $i < $matchCount; $i++)

    1417. 

  1417.                     {

    1418. 

  1418.                         if ($matches[1][$i] != '')

    1419. 

  1419.                         {

    1420. 

  1420.                             $urls[] = $matches[1][$i];

    1421. 

  1421.                         }

    1422. 

  1422.                     }

    1423. 

  1423.                     $matchCount = preg_match_all("~window.open\\s*\\(\\s*\"(.+?)\",~i", $allData, $matches);

    1424. 

  1424.                     for ($i = 0; $i < $matchCount; $i++)

    1425. 

  1425.                     {

    1426. 

  1426.                         if ($matches[1][$i] != '')

    1427. 

  1427.                         {

    1428. 

  1428.                             $urls[] = $matches[1][$i];

    1429. 

  1429.                         }

    1430. 

  1430.                     }

    1431. 

  1431.                 }

    1432. 

  1432. 

    1433. 

  1433.                 /* Remove JavaScript and comments */

    1434. 

  1434.                 $followData = preg_replace('#(<SCRIPT[^>]*?>.*?</SCRIPT>)|(<!--.*?-->)#i', '', $followData);

    1435. 

  1435. 

    1436. 

  1436.                 /* Find any double quoted links */

    1437. 

  1437.                 $matchCount = preg_match_all("~<(A|AREA)\\s+([^>]*?\\s+)*?HREF\\s*=\\s*\"\\s*([^>]+?)\\s*\"[^>]*>~i", $followData, $matches);

    1438. 

  1438.                 for ($i = 0; $i < $matchCount; $i++)

    1439. 

  1439.                 {

    1440. 

  1440.                     if (!eregi('rel="nofollow"', $matches[0][$i]))

    1441. 

  1441.                     {

    1442. 

  1442.                         $urls[] = $matches[3][$i];

    1443. 

  1443.                     }

    1444. 

  1444.                 }

    1445. 

  1445.                 

    1446. 

  1446.                 /* Find any single quoted links */

    1447. 

  1447.                 $matchCount = preg_match_all("~<(A|AREA)\\s+([^>]*?\\s+)*?HREF\\s*=\\s*'\\s*([^>]+?)\\s*'[^>]*>~i", $followData, $matches);

    1448. 

  1448.                 for ($i = 0; $i < $matchCount; $i++)

    1449. 

  1449.                 {

    1450. 

  1450.                     if (!eregi('rel="nofollow"', $matches[0][$i]))

    1451. 

  1451.                     {

    1452. 

  1452.                         $urls[] = $matches[3][$i];

    1453. 

  1453.                     }

    1454. 

  1454.                 }

    1455. 

  1455. 

    1456. 

  1456.                 if ($isearch_config['follow_frames'])

    1457. 

  1457.                 {

    1458. 

  1458.                     /* Search for frames, and add the referenced docs to the URL list */

    1459. 

  1459.                     $matchCount = preg_match_all("~<FRAME\\s+(.*?)>~i", $allData, $matches);

    1460. 

  1460.                     for ($i = 0; $i < $matchCount; $i++)

    1461. 

  1461.                     {

    1462. 

  1462.                         $matchCount2 = preg_match_all("~SRC\\s*=\\s*\"\\s*([^>]+?)\\s*\"~i", $matches[1][$i], $matches2);

    1463. 

  1463.                         for ($j = 0; $j < $matchCount2; $j++)

    1464. 

  1464.                         {

    1465. 

  1465.                             $urls[] = $matches2[1][$j];

    1466. 

  1466.                         }

    1467. 

  1467.                     }

    1468. 

  1468.                 }

    1469. 

  1469. 

    1470. 

  1470.                 $lastUrl = '';

    1471. 

  1471.                 sort($urls);

    1472. 

  1472.                 foreach ($urls as $url)

    1473. 

  1473.                 {

    1474. 

  1474.                     $decodedUrl = str_replace(' ', '%20', ereg_replace('#.*$', '', urldecode(trim($url))));

    1475. 

  1475.                     if (($decodedUrl != $lastUrl) && (!eregi('^javascript:', $decodedUrl)))

    1476. 

  1476.                     {

    1477. 

  1477.                         isearch_addUrl(isearch_relativeToAbsoluteUrl($decodedUrl, $isearch_base), $itemUrl->id);

    1478. 

  1478.                         $lastUrl = $decodedUrl;

    1479. 

  1479.                     }

    1480. 

  1480.                 }

    1481. 

  1481.             }

    1482. 

  1482. 

    1483. 

  1483.             if ($isearch_config['test_mode'] == 1)

    1484. 

  1484.             {

    1485. 

  1485.                 $newState = 'ok';

    1486. 

  1486.             }

    1487. 

  1487.             else if ($index)

    1488. 

  1488.             {

    1489. 

  1489.                 /* Find a "<TITLE></TITLE>" tag in the document head */

    1490. 

  1490.                 if (preg_match("#<TITLE>\\s*(.*?)\\s*</TITLE>#i", $headData, $matches) == 1)

    1491. 

  1491.                 {

    1492. 

  1492.                     $title = $matches[1];

    1493. 

  1493.                 }

    1494. 

  1494.                 else

    1495. 

  1495.                 {

    1496. 

  1496.                     $title = $itemUrl->url;

    1497. 

  1497.                 }

    1498. 

  1498. 

    1499. 

  1499.                 /* Replace breaking tags and other special chars with spaces */

    1500. 

  1500.                 $bodyData = eregi_replace('(<(hr|br|p|td|th|li)(>| [^>]*>))|(&(nb|em|en)sp;?)', ' ', $bodyData);

    1501. 

  1501. 

    1502. 

  1502.                 /* Replace dash entities with dashes */

    1503. 

  1503.                 $bodyData = eregi_replace('&([nm]dash|shy);?', '-', $bodyData);

    1504. 

  1504. 

    1505. 

  1505.                 /* Replace other entities */

    1506. 

  1506.                 $bodyData = eregi_replace('&[lr]squo;?', "'", $bodyData);

    1507. 

  1507.                 $bodyData = eregi_replace('&[lr]dquo;?', '"', $bodyData);

    1508. 

  1508. 

    1509. 

  1509.                 $tdata1 = spliti('(<!-- ISEARCH_BEGIN_INDEX -->|</noindex>)', ' ' . $bodyData);

    1510. 

  1510.                 if (count($tdata1) > 1)

    1511. 

  1511.                 {

    1512. 

  1512.                     /* At least 1 found. */

    1513. 

  1513. 

    1514. 

  1514.                     $bodyData = '';

    1515. 

  1515. 

    1516. 

  1516.                     /* Check for an END_INDEX before the first BEGIN_INDEX */

    1517. 

  1517.                     $tdata2 = spliti('(<!-- ISEARCH_END_INDEX -->|<noindex>)', $tdata1[0], 2);

    1518. 

  1518.                     if (count($tdata2) > 1)

    1519. 

  1519.                     {

    1520. 

  1520.                         /* And END_INDEX was found. Add anything before it into the bodyData */

    1521. 

  1521.                         $bodyData .= $tdata2[0];

    1522. 

  1522.                     }

    1523. 

  1523. 

    1524. 

  1524.                     for ($i = 1; $i < count($tdata1); $i++)

    1525. 

  1525.                     {

    1526. 

  1526.                         $tdata2 = spliti('(<!-- ISEARCH_END_INDEX -->|<noindex>)', $tdata1[$i], 2);

    1527. 

  1527.                         $bodyData .= $tdata2[0];

    1528. 

  1528.                     }

    1529. 

  1529.                 }

    1530. 

  1530.                 else

    1531. 

  1531.                 {

    1532. 

  1532.                     $tdata2 = spliti('(<!-- ISEARCH_END_INDEX -->|<noindex>)', $tdata1[0], 2);

    1533. 

  1533.                     if (count($tdata2) > 1)

    1534. 

  1534.                     {

    1535. 

  1535.                         /* An END_INDEX was found. Add anything before it into the bodyData */

    1536. 

  1536.                         $bodyData = $tdata2[0];

    1537. 

  1537.                     }

    1538. 

  1538.                 }

    1539. 

  1539. 

    1540. 

  1540.                 /* Strip out JavaScript, HTML comments and embedded CSS

    1541. 

  1541.                  */

    1542. 

  1542.                 $bodyData = preg_replace('#(<SCRIPT[^>]*?>.*?</SCRIPT>)|(<!--.*?-->)|(<STYLE[^>]*?>.*?</STYLE>)#i', '', $bodyData);

    1543. 

  1543. 

    1544. 

  1544.                 /* Strip out all HTML tags except special ones */

    1545. 

  1545.                 $bodyData = strip_tags($bodyData, '<h1><h2><h3><h4><h5><img>');

    1546. 

  1546. 

    1547. 

  1547.                 if (! $isearch_config['ignore_image_alt_tags'])

    1548. 

  1548.                 {

    1549. 

  1549.                     /* Replace images with their alt text */

    1550. 

  1550.                     $bodyData = preg_replace('#<IMG\\s[^>]*?ALT\\s*=\\s*("|\')(.*?)\\1.*?>#i', ' \\2 ', $bodyData);

    1551. 

  1551.                 }

    1552. 

  1552. 

    1553. 

  1553.                 $bigWords = '';

    1554. 

  1554. 

    1555. 

  1555.                 $matchCount = preg_match_all("~<H[1-5]>\\s*(.*?)\\s*</H[1-5]>>~i", $bodyData, $matches);

    1556. 

  1556.                 for ($i = 0; $i < $matchCount; $i++)

    1557. 

  1557.                 {

    1558. 

  1558.                     $bigWords .= ' ' . $matches[1][$i];

    1559. 

  1559.                 }

    1560. 

  1560. 

    1561. 

  1561.                 /* Translate from HTML to ASCII */

    1562. 

  1562.                 $bodyData = isearch_html_entity_decode($bodyData, ENT_QUOTES, $isearch_config['char_set']);

    1563. 

  1563.                 $bigWords = isearch_html_entity_decode($bigWords, ENT_QUOTES, $isearch_config['char_set']);

    1564. 

  1564.                 $keyWords = isearch_html_entity_decode($keyWords, ENT_QUOTES, $isearch_config['char_set']);

    1565. 

  1565.                 $titleWords = isearch_html_entity_decode($title, ENT_QUOTES, $isearch_config['char_set']);

    1566. 

  1566. 

    1567. 

  1567.                 /* Keep stripped copy of document body */

    1568. 

  1568.                 $strippedBody = preg_replace("/\\s+/", ' ', strip_tags($bodyData));

    1569. 

  1569. 

    1570. 

  1570.                 /* Strip out unwanted characters from the strings that we

    1571. 

  1571.                  * search on

    1572. 

  1572.                  */

    1573. 

  1573.                 $bodyData = isearch_cleanString($bodyData, $charset);

    1574. 

  1574.                 $bigWords = isearch_cleanString($bigWords, $charset);

    1575. 

  1575.                 $keyWords = isearch_cleanString($keyWords, $charset);

    1576. 

  1576.                 $titleWords = isearch_cleanString($titleWords, $charset);

    1577. 

  1577. 

    1578. 

  1578.                 $urlWords = ereg_replace('((https?://)|[+/\\\\\\.]|(%20))', ' ', $itemUrl->url);

    1579. 

  1579.                 $urlWords = isearch_cleanString($urlWords, $charset);

    1580. 

  1580. 

    1581. 

  1581.                 $score = array();

    1582. 

  1582.                 for ($i = 0; $i < 5; $i++)

    1583. 

  1583.                 {

    1584. 

  1584.                     if ($i == 0)

    1585. 

  1585.                     {

    1586. 

  1586.                         $words = explode(' ', $bodyData);

    1587. 

  1587.                         $wordScore = $isearch_config['word_rank'];

    1588. 

  1588.                     }

    1589. 

  1589.                     else if ($i == 1)

    1590. 

  1590.                     {

    1591. 

  1591.                         $words = explode(' ', $bigWords);

    1592. 

  1592.                         $wordScore = $isearch_config['heading_rank'];

    1593. 

  1593.                     }

    1594. 

  1594.                     else if ($i == 2)

    1595. 

  1595.                     {

    1596. 

  1596.                         $words = explode(' ', $keyWords);

    1597. 

  1597.                         $wordScore = $isearch_config['keyword_rank'];

    1598. 

  1598.                     }

    1599. 

  1599.                     else if ($i == 3)

    1600. 

  1600.                     {

    1601. 

  1601.                         $words = explode(' ', $titleWords);

    1602. 

  1602.                         $wordScore = $isearch_config['title_rank'];

    1603. 

  1603.                     }

    1604. 

  1604.                     else if ($i == 4)

    1605. 

  1605.                     {

    1606. 

  1606.                         $words = explode(' ', $urlWords);

    1607. 

  1607.                         $wordScore = $isearch_config['url_rank'];

    1608. 

  1608.                     }

    1609. 

  1609. 

    1610. 

  1610.                     if ($wordScore == 0)

    1611. 

  1611.                     {

    1612. 

  1612.                         // Skip if wordScore is zero

    1613. 

  1613.                         continue;

    1614. 

  1614.                     }

    1615. 

  1615. 

    1616. 

  1616.                     foreach ($words as $word)

    1617. 

  1617.                     {

    1618. 

  1618.                         if (($word != '') && (!in_array($word, $isearch_config['stop_words'])) && (strlen($word) > $isearch_config['stop_words_length']))

    1619. 

  1619.                         {

    1620. 

  1620.                             if (isset($score[$word]))

    1621. 

  1621.                             {

    1622. 

  1622.                                 $score[$word] += $wordScore;

    1623. 

  1623.                             }

    1624. 

  1624.                             else

    1625. 

  1625.                             {

    1626. 

  1626.                                 $score[$word] = $wordScore;

    1627. 

  1627.                             }

    1628. 

  1628.                         }

    1629. 

  1629.                     }

    1630. 

  1630.                 }

    1631. 

  1631. 

    1632. 

  1632.                 if (count($score) == 0)

    1633. 

  1633.                 {

    1634. 

  1634.                     isearch_log('WARNING: No words were found', 4);

    1635. 

  1635.                 }

    1636. 

  1636.                 else

    1637. 

  1637.                 {

    1638. 

  1638.                     $query = "INSERT INTO $isearch_table_words_new (word, id, score) VALUES ";

    1639. 

  1639.                     $needComma = False;

    1640. 

  1640.                     $id = $itemUrl->id;

    1641. 

  1641.                     foreach (array_keys($score) as $word)

    1642. 

  1642.                     {

    1643. 

  1643.                         if ($needComma)

    1644. 

  1644.                         {

    1645. 

  1645.                             $query .= ',';

    1646. 

  1646.                         }

    1647. 

  1647.                         $needComma = True;

    1648. 

  1648.                         $query .= "('".isearch_escape_string($word)."', '$id', '".$score[$word]."')";

    1649. 

  1649.                     }

    1650. 

  1650.                     if (!mysql_query($query, $isearch_db))

    1651. 

  1651.                     {

    1652. 

  1652.                         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1653. 

  1653.                     }

    1654. 

  1654.                 }

    1655. 

  1655. 

    1656. 

  1656.                 /* Update this URL's details in the database */

    1657. 

  1657.                 if (!mysql_query("UPDATE $isearch_table_urls_new SET " .

    1658. 

  1658.                                  "title='" . isearch_escape_string($title) . "', " .

    1659. 

  1659.                                  "description='" . isearch_escape_string($description) . "', " .

    1660. 

  1660.                                  "cache='" . isearch_escape_string($cache) . "', " .

    1661. 

  1661.                                  "stripped_body='" . isearch_escape_string($strippedBody) . "', " .

    1662. 

  1662.                                  "words=' " . isearch_escape_string($bodyData) . " ', " .

    1663. 

  1663.                                  "size='$size', " .

    1664. 

  1664.                                  "base='" . isearch_escape_string($isearch_base) . "', " .

    1665. 

  1665.                                  "priority='$priority', " .

    1666. 

  1666.                                  "changefreq='" . isearch_escape_string($changefreq) . "', " .

    1667. 

  1667.                                  "lastmod='$lastModified', " .

    1668. 

  1668.                                  "sig='" . isearch_escape_string($sig) . "' " .

    1669. 

  1669.                                  "WHERE id='" . $itemUrl->id . "'", $isearch_db))

    1670. 

  1670.                 {

    1671. 

  1671.                     isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1672. 

  1672.                 }

    1673. 

  1673.                 $newState = 'ok';

    1674. 

  1674.             }

    1675. 

  1675.             else    /* if ($index) */

    1676. 

  1676.             {

    1677. 

  1677.                  $newState = 'noindex';

    1678. 

  1678.             }

    1679. 

  1679.         }

    1680. 

  1680.     }

    1681. 

  1681. 

    1682. 

  1682.     if (!mysql_query("UPDATE $isearch_table_urls_new SET state='$newState' WHERE id='" . $itemUrl->id . "'", $isearch_db))

    1683. 

  1683.     {

    1684. 

  1684.         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1685. 

  1685.     }

    1686. 

  1686. 

    1687. 

  1687.     return True;

    1688. 

  1688. }

    1689. 

  1689. 

    1690. 

  1690. 

    1691. 

  1691. /* Glue the fragments from a parse_url together. */

    1692. 

  1692. function glue_url($parsed)

    1693. 

  1693. {

    1694. 

  1694.     if (! is_array($parsed))

    1695. 

  1695.     {

    1696. 

  1696.         return false;

    1697. 

  1697.     }

    1698. 

  1698. 

    1699. 

  1699.     if (isset($parsed['scheme']))

    1700. 

  1700.     {

    1701. 

  1701.         $sep = (strtolower($parsed['scheme']) == 'mailto' ? ':' : '://');

    1702. 

  1702.         $uri = $parsed['scheme'] . $sep;

    1703. 

  1703.     }

    1704. 

  1704.     else

    1705. 

  1705.     {

    1706. 

  1706.         $uri = '';

    1707. 

  1707.     }

    1708. 

  1708. 

    1709. 

  1709.     if (isset($parsed['pass']))

    1710. 

  1710.     {

    1711. 

  1711.         $uri .= "$parsed[user]:$parsed[pass]@";

    1712. 

  1712.     }

    1713. 

  1713.     elseif (isset($parsed['user']))

    1714. 

  1714.     {

    1715. 

  1715.         $uri .= "$parsed[user]@";

    1716. 

  1716.     }

    1717. 

  1717. 

    1718. 

  1718.     if (isset($parsed['host']))

    1719. 

  1719.     {

    1720. 

  1720.         $uri .= $parsed['host'];

    1721. 

  1721.     }

    1722. 

  1722.     if (isset($parsed['port']))

    1723. 

  1723.     {

    1724. 

  1724.         $uri .= ":$parsed[port]";

    1725. 

  1725.     }

    1726. 

  1726.     if (isset($parsed['path']))

    1727. 

  1727.     {

    1728. 

  1728.         $uri .= $parsed['path'];

    1729. 

  1729.     }

    1730. 

  1730.     if (isset($parsed['query']))

    1731. 

  1731.     {

    1732. 

  1732.         $uri .= "?$parsed[query]";

    1733. 

  1733.     }

    1734. 

  1734.     if (isset($parsed['fragment']))

    1735. 

  1735.     {

    1736. 

  1736.         $uri .= "#$parsed[fragment]";

    1737. 

  1737.     }

    1738. 

  1738. 

    1739. 

  1739.     return $uri;

    1740. 

  1740. }

    1741. 

  1741. 

    1742. 

  1742. 

    1743. 

  1743. /* Add a URL to the search index */

    1744. 

  1744. function isearch_addUrl($absoluteUrl, $referrer_id)

    1745. 

  1745. {

    1746. 

  1746.     global $isearch_table_urls_new;

    1747. 

  1747.     global $isearch_db;

    1748. 

  1748. 

    1749. 

  1749.     global $isearch_config;

    1750. 

  1750. 

    1751. 

  1751.     if (eregi('^mailto:', $absoluteUrl))

    1752. 

  1752.     {

    1753. 

  1753.         isearch_log("INFO:  Ignoring URL [$absoluteUrl]", 6);

    1754. 

  1754.         return;

    1755. 

  1755.     }

    1756. 

  1756.     isearch_log("INFO:  Checking URL [$absoluteUrl]", 7);

    1757. 

  1757. 

    1758. 

  1758.     /* Split the absolute URL into component parts */

    1759. 

  1759.     $absoluteParts = @parse_url($absoluteUrl);

    1760. 

  1760. 

    1761. 

  1761.     if ((!isset($absoluteParts['scheme'])) || (!isset($absoluteParts['host'])))

    1762. 

  1762.     {

    1763. 

  1763.         /* Unable to parse URL */

    1764. 

  1764.         isearch_log("INFO:    Unable to parse absoluteUrl [$absoluteUrl]", 4);

    1765. 

  1765.         return;

    1766. 

  1766.     }

    1767. 

  1767. 

    1768. 

  1768.     $absoluteParts['scheme'] = strtolower($absoluteParts['scheme']);

    1769. 

  1769.     if (!ereg('^(https?|ftps?)$', $absoluteParts['scheme']))

    1770. 

  1770.     {

    1771. 

  1771.         isearch_log("INFO:    Rejecting unsupported URL format [$absoluteUrl]", 6);

    1772. 

  1772.         return;

    1773. 

  1773.     }

    1774. 

  1774. 

    1775. 

  1775.     /* Remove any anchor reference in the URL */

    1776. 

  1776.     unset($absoluteParts['fragment']);

    1777. 

  1777. 

    1778. 

  1778.     /* Split the GET variables and remove unnecessary ones */

    1779. 

  1779.     if ((count($isearch_config['remove_get_vars']) == 1) && ($isearch_config['remove_get_vars'][0] == '*'))

    1780. 

  1780.     {

    1781. 

  1781.         /* Strip all GET variables */

    1782. 

  1782.         unset($absoluteParts['query']);

    1783. 

  1783.     }

    1784. 

  1784.     else if (isset($absoluteParts['query']))

    1785. 

  1785.     {

    1786. 

  1786.         $queryParts = explode('&', $absoluteParts['query']);

    1787. 

  1787. 

    1788. 

  1788.         foreach (array_keys($queryParts) as $key)

    1789. 

  1789.         {

    1790. 

  1790.             $varName = eregi_replace('=.*$', '', $queryParts[$key]);

    1791. 

  1791.             if (in_array($varName, $isearch_config['remove_get_vars']))

    1792. 

  1792.             {

    1793. 

  1793.                 unset($queryParts[$key]);

    1794. 

  1794.             }

    1795. 

  1795.         }

    1796. 

  1796. 

    1797. 

  1797.         $absoluteParts['query'] = implode('&', $queryParts);

    1798. 

  1798. 

    1799. 

  1799.         if ($absoluteParts['query'] == '')

    1800. 

  1800.         {

    1801. 

  1801.             unset($absoluteParts['query']);

    1802. 

  1802.         }

    1803. 

  1803.     }

    1804. 

  1804. 

    1805. 

  1805.     /* Handle the www subdomain according to the value of $isearch_config['www_option'].

    1806. 

  1806.      *  1 = Leave as is

    1807. 

  1807.      *  2 = Strip www subdomains

    1808. 

  1808.      *  3 = Add www subdomain

    1809. 

  1809.      */

    1810. 

  1810.     if ($isearch_config['www_option'] == 2)

    1811. 

  1811.     {

    1812. 

  1812.         $absoluteParts['host'] = eregi_replace('^www\.', '', $absoluteParts['host']);

    1813. 

  1813.     }

    1814. 

  1814.     else if ($isearch_config['www_option'] == 3)

    1815. 

  1815.     {

    1816. 

  1816.         if (!eregi('^www\.', $absoluteParts['host']))

    1817. 

  1817.         {

    1818. 

  1818.             $absoluteParts['host'] = 'www.' . $absoluteParts['host'];

    1819. 

  1819.         }

    1820. 

  1820.     }

    1821. 

  1821. 

    1822. 

  1822.     /* Replace space characters with %20 */

    1823. 

  1823.     if (isset($absoluteParts['path']))

    1824. 

  1824.     {

    1825. 

  1825.         $absoluteParts['path'] = str_replace(' ', '%20', $absoluteParts['path']);

    1826. 

  1826.     }

    1827. 

  1827. 

    1828. 

  1828.     /* Glue URL parts together again */

    1829. 

  1829.     $absoluteUrl = glue_url($absoluteParts);

    1830. 

  1830. 

    1831. 

  1831.     if (isset($absoluteParts['path']))

    1832. 

  1832.     {

    1833. 

  1833.         $fileName = eregi_replace('.*/', '', $absoluteParts['path']);

    1834. 

  1834.         $temp = explode('.', $fileName);

    1835. 

  1835.         if (count($temp) < 2)

    1836. 

  1836.         {

    1837. 

  1837.             $fileExtension = '';

    1838. 

  1838.         }

    1839. 

  1839.         else

    1840. 

  1840.         {

    1841. 

  1841.             $fileExtension = strtolower($temp[count($temp)-1]);

    1842. 

  1842.         }

    1843. 

  1843.     }

    1844. 

  1844.     else

    1845. 

  1845.     {

    1846. 

  1846.         $fileExtension = '';

    1847. 

  1847.         $fileName = '/';

    1848. 

  1848.     }

    1849. 

  1849. 

    1850. 

  1850.     isearch_log("INFO:    Absolute URL [$absoluteUrl]", 6);

    1851. 

  1851. 

    1852. 

  1852.     $allowed = False;

    1853. 

  1853.     if ($fileExtension == '')

    1854. 

  1854.     {

    1855. 

  1855.         /* Treat no file extension as a directory. */

    1856. 

  1856.         if ($isearch_config['directory_handling'] != 0)

    1857. 

  1857.         {

    1858. 

  1858.             /* Directories are allowed */

    1859. 

  1859.             $allowed = True;

    1860. 

  1860. 

    1861. 

  1861.             if ($isearch_config['directory_handling'] == 2)

    1862. 

  1862.             {

    1863. 

  1863.                 /* Add trailing slash to directories */

    1864. 

  1864.                 $absoluteUrl = ereg_replace('([^/])$', '\1/', $absoluteUrl);

    1865. 

  1865.             }

    1866. 

  1866.             else if ($isearch_config['directory_handling'] == 3)

    1867. 

  1867.             {

    1868. 

  1868.                 /* Strip trailing slash from directories */

    1869. 

  1869.                 $absoluteUrl = ereg_replace('/$', '', $absoluteUrl);

    1870. 

  1870.             }

    1871. 

  1871.         }

    1872. 

  1872.     }

    1873. 

  1873.     else

    1874. 

  1874.     {

    1875. 

  1875.         /* Check whether this is a default file name, and strip it if so. */

    1876. 

  1876.         foreach ($isearch_config['strip_defaults'] as $item)

    1877. 

  1877.         {

    1878. 

  1878.             if (($item != '') && ($item == $fileName))

    1879. 

  1879.             {

    1880. 

  1880.                 $absoluteUrl = str_replace("/$item", '/', $absoluteUrl);

    1881. 

  1881.                 $allowed = True;

    1882. 

  1882.                 break;

    1883. 

  1883.             }

    1884. 

  1884.         }

    1885. 

  1885. 

    1886. 

  1886.         /* Check whether this file extension is allowed. Always allow PDF files

    1887. 

  1887.          * when PDF is enabled and DOC files when Word is enabled.

    1888. 

  1888.          */

    1889. 

  1889.         if ((in_array($fileExtension, $isearch_config['allowed_ext'])) ||

    1890. 

  1890.             (($isearch_config['pdf_support'] != 0) && ($fileExtension == 'pdf')) ||

    1891. 

  1891.             (($isearch_config['msword_support'] != 0) && ($fileExtension == 'doc')))

    1892. 

  1892.         {

    1893. 

  1893.             $allowed = True;

    1894. 

  1894.         }

    1895. 

  1895.     }

    1896. 

  1896. 

    1897. 

  1897.     if (!$allowed)

    1898. 

  1898.     {

    1899. 

  1899.         isearch_log("INFO:    Rejecting URL - it is not in allowed file extensions [$absoluteUrl]", 5);

    1900. 

  1900.     }

    1901. 

  1901.     else

    1902. 

  1902.     {

    1903. 

  1903.         /* Check whether it's already in the table of known URLs */

    1904. 

  1904.         $result = mysql_query("SELECT state, id FROM $isearch_table_urls_new WHERE url='".isearch_escape_string($absoluteUrl)."'", $isearch_db);

    1905. 

  1905.         if (!$result)

    1906. 

  1906.         {

    1907. 

  1907.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1908. 

  1908.         }

    1909. 

  1909.         else if (mysql_num_rows($result) > 0)

    1910. 

  1910.         {

    1911. 

  1911.             /* Already in database */

    1912. 

  1912.             isearch_log("INFO:    URL allowed, but already in database [$absoluteUrl]", 6);

    1913. 

  1913.             if ($item = mysql_fetch_object($result))

    1914. 

  1914.             {

    1915. 

  1915.                 if (($item->state == 'new') && ($item->id != $referrer_id))

    1916. 

  1916.                 {

    1917. 

  1917.                     if (!mysql_query("UPDATE $isearch_table_urls_new SET temp_referrer_id='$referrer_id' WHERE url='".isearch_escape_string($absoluteUrl)."'", $isearch_db))

    1918. 

  1918.                     {

    1919. 

  1919.                         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    1920. 

  1920.                     }

    1921. 

  1921.                 }

    1922. 

  1922.             }

    1923. 

  1923.             return;

    1924. 

  1924.         }

    1925. 

  1925. 

    1926. 

  1926.         $allowed = False;

    1927. 

  1927. 

    1928. 

  1928.         /* Check that this URL is in the allowed list */

    1929. 

  1929.         foreach (isearch_expandList($isearch_config['allowed_urls_beginning']) as $item)

    1930. 

  1930.         {

    1931. 

  1931.             if (($item != '') && ($item == substr($absoluteUrl, 0, strlen($item))))

    1932. 

  1932.             {

    1933. 

  1933.                 $allowed = True;

    1934. 

  1934.                 break;

    1935. 

  1935.             }

    1936. 

  1936.         }

    1937. 

  1937. 

    1938. 

  1938.         foreach (isearch_expandList($isearch_config['allowed_urls']) as $item)

    1939. 

  1939.         {

    1940. 

  1940.             if (($item != '') && (eregi($item, $absoluteUrl)))

    1941. 

  1941.             {

    1942. 

  1942.                 $allowed = True;

    1943. 

  1943.                 break;

    1944. 

  1944.             }

    1945. 

  1945.         }

    1946. 

  1946. 

    1947. 

  1947.         if (!$allowed)

    1948. 

  1948.         {

    1949. 

  1949.             isearch_log("INFO:    Rejecting URL - it  is not in allowed URL list [$absoluteUrl]", 5);

    1950. 

  1950.         }

    1951. 

  1951.         else

    1952. 

  1952.         {

    1953. 

  1953.             if (($absoluteParts['scheme'] == 'http') && (!in_array($absoluteParts['host'], $isearch_config['robots_domains'])))

    1954. 

  1954.             {

    1955. 

  1955.                 /* Parse the robots.txt for this domain */

    1956. 

  1956.                 $host = $absoluteParts['host'];

    1957. 

  1957.                 if (isset($absoluteParts['port']))

    1958. 

  1958.                 {

    1959. 

  1959.                     $host .= ':'.$absoluteParts['port'];

    1960. 

  1960.                 }

    1961. 

  1961.                 isearch_parseRobots($host);

    1962. 

  1962.             }

    1963. 

  1963. 

    1964. 

  1964.             /* Check that this URL is not in the disallowed list */

    1965. 

  1965.             foreach (isearch_expandList($isearch_config['exclude_urls_beginning']) as $item)

    1966. 

  1966.             {

    1967. 

  1967.                 if (($item != '') && ($item == substr($absoluteUrl, 0, strlen($item))))

    1968. 

  1968.                 {

    1969. 

  1969.                     $allowed = False;

    1970. 

  1970.                     isearch_log("INFO:    Rejecting URL - it is in disallowed URL beginning list [$absoluteUrl]", 5);

    1971. 

  1971.                     break;

    1972. 

  1972.                 }

    1973. 

  1973.             }

    1974. 

  1974. 

    1975. 

  1975.             if ($allowed)

    1976. 

  1976.             {

    1977. 

  1977.                 foreach (isearch_expandList($isearch_config['exclude_urls']) as $item)

    1978. 

  1978.                 {

    1979. 

  1979.                     if (($item != '') && (eregi($item, $absoluteUrl)))

    1980. 

  1980.                     {

    1981. 

  1981.                         $allowed = False;

    1982. 

  1982.                         isearch_log("INFO:    Rejecting URL - it is in disallowed URL regexp list [$absoluteUrl]", 5);

    1983. 

  1983.                         break;

    1984. 

  1984.                     }

    1985. 

  1985.                 }

    1986. 

  1986.             }

    1987. 

  1987. 

    1988. 

  1988.             if ($allowed)

    1989. 

  1989.             {

    1990. 

  1990.                 /* Check the robots.txt excludes list */

    1991. 

  1991.                 foreach ($isearch_config['robots_excludes'] as $item)

    1992. 

  1992.                 {

    1993. 

  1993.                     if (($item != '') && (eregi($item, $absoluteUrl)))

    1994. 

  1994.                     {

    1995. 

  1995.                         $allowed = False;

    1996. 

  1996.                         isearch_log("INFO:    Rejecting URL - it is disallowed by robots.txt [$absoluteUrl]", 5);

    1997. 

  1997.                         break;

    1998. 

  1998.                     }

    1999. 

  1999.                 }

    2000. 

  2000.             }

    2001. 

  2001.         }

    2002. 

  2002.     }

    2003. 

  2003. 

    2004. 

  2004.     if ($allowed)

    2005. 

  2005.     {

    2006. 

  2006.         /* Add it */

    2007. 

  2007.         if (!mysql_query("INSERT INTO $isearch_table_urls_new (url, temp_referrer_id, state) VALUES ('".isearch_escape_string($absoluteUrl)."', '$referrer_id', 'new')", $isearch_db))

    2008. 

  2008.         {

    2009. 

  2009.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2010. 

  2010.         }

    2011. 

  2011.         isearch_log("INFO:    URL allowed, and added [$absoluteUrl]", 4);

    2012. 

  2012.     }

    2013. 

  2013. }

    2014. 

  2014. 

    2015. 

  2015. 

    2016. 

  2016. /* Expand a list of URLs. If an entry begins with an "@" it is treated as a

    2017. 

  2017.  * filename containing more URLs.

    2018. 

  2018.  */

    2019. 

  2019. function isearch_expandList($list, $depth=0)

    2020. 

  2020. {

    2021. 

  2021.     $items = array();

    2022. 

  2022.     if ($depth > 20)

    2023. 

  2023.     {

    2024. 

  2024.         isearch_log('ERROR: Too much recursion in isearch_expandUrlList', 1);

    2025. 

  2025.         return $items;

    2026. 

  2026.     }

    2027. 

  2027. 

    2028. 

  2028.     foreach ($list as $item)

    2029. 

  2029.     {

    2030. 

  2030.         if ($item{0} == '@')

    2031. 

  2031.         {

    2032. 

  2032.             $lines = @file(substr($item, 1));

    2033. 

  2033.             if (is_array($lines))

    2034. 

  2034.             {

    2035. 

  2035.                 $newList = array();

    2036. 

  2036.                 foreach ($lines as $line)

    2037. 

  2037.                 {

    2038. 

  2038.                     $newList[] = trim($line);

    2039. 

  2039.                 }

    2040. 

  2040.                 $items = array_merge($items, isearch_expandList($newList, $depth + 1));

    2041. 

  2041.             }

    2042. 

  2042.             else

    2043. 

  2043.             {

    2044. 

  2044.                 isearch_log('WARNING: unable to read file : ' . substr($item, 1));

    2045. 

  2045.             }

    2046. 

  2046.         }

    2047. 

  2047.         else

    2048. 

  2048.         {

    2049. 

  2049.             $items[] = $item;

    2050. 

  2050.         }

    2051. 

  2051.     }

    2052. 

  2052.     return $items;

    2053. 

  2053. }

    2054. 

  2054. 

    2055. 

  2055. 

    2056. 

  2056. /* Reset the search index to allow site to be re-spidered */

    2057. 

  2057. function isearch_reset()

    2058. 

  2058. {

    2059. 

  2059.     global $isearch_table_urls_new, $isearch_table_info, $isearch_table_words_new;

    2060. 

  2060.     global $isearch_db;

    2061. 

  2061. 

    2062. 

  2062.     global $isearch_config;

    2063. 

  2063. 

    2064. 

  2064.     /* Clear the spider log */

    2065. 

  2065.     isearch_clearLog();

    2066. 

  2066. 

    2067. 

  2067.     isearch_log('INFO: Starting spidering ' . date('dS F Y h:i:s A'), 3);

    2068. 

  2068. 

    2069. 

  2069.     /* Delete all entries from the new databases */

    2070. 

  2070.     if (!mysql_query("DELETE FROM $isearch_table_urls_new", $isearch_db))

    2071. 

  2071.     {

    2072. 

  2072.         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2073. 

  2073.     }

    2074. 

  2074. 

    2075. 

  2075.     if (!mysql_query("DELETE FROM $isearch_table_words_new", $isearch_db))

    2076. 

  2076.     {

    2077. 

  2077.         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2078. 

  2078.     }

    2079. 

  2079. 

    2080. 

  2080.     $urls = isearch_expandList($isearch_config['start_urls']);

    2081. 

  2081. 

    2082. 

  2082.     /* Add all start URLs to database */

    2083. 

  2083.     foreach ($urls as $url)

    2084. 

  2084.     {

    2085. 

  2085.         /* Find whether this URL refers to a directory */

    2086. 

  2086.         $urlParts = @parse_url($url);

    2087. 

  2087.         if ((!isset($urlParts['scheme'])) || (!isset($urlParts['host'])))

    2088. 

  2088.         {

    2089. 

  2089.             /* Unable to parse URL */

    2090. 

  2090.             isearch_log("ERROR: Unable to parse start URL [$url]", 1);

    2091. 

  2091.         }

    2092. 

  2092.         else

    2093. 

  2093.         {

    2094. 

  2094.             $filePath = isset($urlParts['path']) ? $urlParts['path'] : '';

    2095. 

  2095.             $fileName = eregi_replace('.*/', '', $filePath);

    2096. 

  2096.             $temp = explode('.', $fileName);

    2097. 

  2097.             if (count($temp) < 2)

    2098. 

  2098.             {

    2099. 

  2099.                 $fileExtension = '';

    2100. 

  2100.             }

    2101. 

  2101.             else

    2102. 

  2102.             {

    2103. 

  2103.                 $fileExtension = $temp[count($temp)-1];

    2104. 

  2104.             }

    2105. 

  2105. 

    2106. 

  2106.             if ($fileExtension == '')

    2107. 

  2107.             {

    2108. 

  2108.                 /* Treat no file extension as a directory. Check whether there is a

    2109. 

  2109.                  * trailing slash on the URL and add it if necessary.

    2110. 

  2110.                  */

    2111. 

  2111.                 if ($isearch_config['directory_handling'] == 2)

    2112. 

  2112.                 {

    2113. 

  2113.                     /* Add trailing slash to directories */

    2114. 

  2114.                     $url = ereg_replace('([^/])$', '\1/', $url);

    2115. 

  2115.                 }

    2116. 

  2116.                 else if ($isearch_config['directory_handling'] == 3)

    2117. 

  2117.                 {

    2118. 

  2118.                     /* Strip trailing slash from directories */

    2119. 

  2119.                     $url = ereg_replace('/$', '', $url);

    2120. 

  2120.                 }

    2121. 

  2121.             }

    2122. 

  2122. 

    2123. 

  2123.             if (!mysql_query("INSERT INTO $isearch_table_urls_new (url, temp_referrer_id, state) VALUES ('".isearch_escape_string($url)."', '-1', 'new')", $isearch_db))

    2124. 

  2124.             {

    2125. 

  2125.                 isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2126. 

  2126.             }

    2127. 

  2127.             isearch_log("INFO: Added start URL [$url]", 5);

    2128. 

  2128.         }

    2129. 

  2129.     }

    2130. 

  2130. 

    2131. 

  2131.     /* Reset the robots.txt exclude list */

    2132. 

  2132.     $isearch_config['robots_domains'] = array();

    2133. 

  2133.     $isearch_config['robots_excludes'] = array();

    2134. 

  2134.     if (!mysql_query("UPDATE $isearch_table_info SET robots_domains='', robots_excludes='' WHERE id='1'", $isearch_db))

    2135. 

  2135.     {

    2136. 

  2136.         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2137. 

  2137.     }

    2138. 

  2138. 

    2139. 

  2139.     /* Clear the last_update time to indicate that we are currently spidering */

    2140. 

  2140.     if (!mysql_query("UPDATE $isearch_table_info SET last_update='0' WHERE id='1'", $isearch_db))

    2141. 

  2141.     {

    2142. 

  2142.         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2143. 

  2143.     }

    2144. 

  2144. }

    2145. 

  2145. 

    2146. 

  2146. 

    2147. 

  2147. /* Get the number of URLs that are in the database with the specified state (or

    2148. 

  2148.  * all if state is empty).

    2149. 

  2149.  */

    2150. 

  2150. function isearch_getUrlCount($new = False, $state = '')

    2151. 

  2151. {

    2152. 

  2152.     global $isearch_table_urls, $isearch_table_urls_new;

    2153. 

  2153.     global $isearch_db;

    2154. 

  2154. 

    2155. 

  2155.     $count = 0;

    2156. 

  2156. 

    2157. 

  2157.     $query = 'SELECT COUNT(*) FROM ' . ($new ? $isearch_table_urls_new : $isearch_table_urls);

    2158. 

  2158. 

    2159. 

  2159.     if ($state != '')

    2160. 

  2160.     {

    2161. 

  2161.         $query .= " WHERE state='$state'";

    2162. 

  2162.     }

    2163. 

  2163.     $result = mysql_query($query, $isearch_db);

    2164. 

  2164.     if (!$result)

    2165. 

  2165.     {

    2166. 

  2166.         isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2167. 

  2167.     }

    2168. 

  2168.     else

    2169. 

  2169.     {

    2170. 

  2170.         $count = mysql_result($result, 0, 0);

    2171. 

  2171.     }

    2172. 

  2172. 

    2173. 

  2173.     return $count;

    2174. 

  2174. }

    2175. 

  2175. 

    2176. 

  2176. 

    2177. 

  2177. /* Copy the current spidered tables to the search tables. This makes them

    2178. 

  2178.  * searchable.

    2179. 

  2179.  */

    2180. 

  2180. function isearch_copyUrlTables($toNew = False)

    2181. 

  2181. {

    2182. 

  2182.     global $isearch_table_urls, $isearch_table_urls_new;

    2183. 

  2183.     global $isearch_table_words, $isearch_table_words_new;

    2184. 

  2184.     global $isearch_db;

    2185. 

  2185. 

    2186. 

  2186.     if ($toNew)

    2187. 

  2187.     {

    2188. 

  2188.         mysql_query("LOCK TABLES $isearch_table_urls_new WRITE, $isearch_table_words_new WRITE,  $isearch_table_urls READ, $isearch_table_words READ", $isearch_db);

    2189. 

  2189.         if (!mysql_query("DELETE FROM $isearch_table_urls_new", $isearch_db))

    2190. 

  2190.         {

    2191. 

  2191.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2192. 

  2192.         }

    2193. 

  2193. 

    2194. 

  2194.         if (!mysql_query("DELETE FROM $isearch_table_words_new", $isearch_db))

    2195. 

  2195.         {

    2196. 

  2196.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2197. 

  2197.         }

    2198. 

  2198. 

    2199. 

  2199.         if (!mysql_query("INSERT INTO $isearch_table_urls_new SELECT * FROM $isearch_table_urls", $isearch_db))

    2200. 

  2200.         {

    2201. 

  2201.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2202. 

  2202.         }

    2203. 

  2203. 

    2204. 

  2204.         if (!mysql_query("INSERT INTO $isearch_table_words_new SELECT * FROM $isearch_table_words", $isearch_db))

    2205. 

  2205.         {

    2206. 

  2206.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2207. 

  2207.         }

    2208. 

  2208.     }

    2209. 

  2209.     else

    2210. 

  2210.     {

    2211. 

  2211.         mysql_query("LOCK TABLES $isearch_table_urls WRITE, $isearch_table_words WRITE,  $isearch_table_urls_new READ, $isearch_table_words_new READ", $isearch_db);

    2212. 

  2212.         if (!mysql_query("DELETE FROM $isearch_table_urls", $isearch_db))

    2213. 

  2213.         {

    2214. 

  2214.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2215. 

  2215.         }

    2216. 

  2216. 

    2217. 

  2217.         if (!mysql_query("DELETE FROM $isearch_table_words", $isearch_db))

    2218. 

  2218.         {

    2219. 

  2219.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2220. 

  2220.         }

    2221. 

  2221. 

    2222. 

  2222.         if (!mysql_query("INSERT INTO $isearch_table_urls SELECT * FROM $isearch_table_urls_new WHERE state='ok'", $isearch_db))

    2223. 

  2223.         {

    2224. 

  2224.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2225. 

  2225.         }

    2226. 

  2226. 

    2227. 

  2227.         if (!mysql_query("INSERT INTO $isearch_table_words SELECT * FROM $isearch_table_words_new", $isearch_db))

    2228. 

  2228.         {

    2229. 

  2229.             isearch_log('ERROR: MySQL error : ' . mysql_error() . " in " . __FILE__ . " line " . __LINE__, 1);

    2230. 

  2230.         }

    2231. 

  2231.     }

    2232. 

  2232.     mysql_query("UNLOCK TABLES", $isearch_db);

    2233. 

  2233. }

    2234. 

  2234. 

    2235. 

  2235. $isearch_url_fopen_detected = (bool) ini_get('allow_url_fopen');

    2236. 

  2236. $isearch_curl_detected = (function_exists('curl_init')) ? True : False;

    2237. 

  2237. $isearch_sockets_detected = (function_exists('fsockopen')) ? True : False;

    2238. 

  2238. 

    2239. 

  2239. ?>

    2240. 

  2240.