PageRenderTime 54ms CodeModel.GetById 22ms RepoModel.GetById 0ms app.codeStats 1ms

/indra/llui/llkeywords.cpp

https://bitbucket.org/lindenlab/viewer-beta/
C++ | 667 lines | 516 code | 80 blank | 71 comment | 125 complexity | d992b4d8b4f4caada6502222898d30e4 MD5 | raw file
Possible License(s): LGPL-2.1
  1. /**
  2. * @file llkeywords.cpp
  3. * @brief Keyword list for LSL
  4. *
  5. * $LicenseInfo:firstyear=2000&license=viewerlgpl$
  6. * Second Life Viewer Source Code
  7. * Copyright (C) 2010, Linden Research, Inc.
  8. *
  9. * This library is free software; you can redistribute it and/or
  10. * modify it under the terms of the GNU Lesser General Public
  11. * License as published by the Free Software Foundation;
  12. * version 2.1 of the License only.
  13. *
  14. * This library is distributed in the hope that it will be useful,
  15. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  17. * Lesser General Public License for more details.
  18. *
  19. * You should have received a copy of the GNU Lesser General Public
  20. * License along with this library; if not, write to the Free Software
  21. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  22. *
  23. * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
  24. * $/LicenseInfo$
  25. */
  26. #include "linden_common.h"
  27. #include <iostream>
  28. #include <fstream>
  29. #include "llkeywords.h"
  30. #include "lltexteditor.h"
  31. #include "llstl.h"
  32. #include <boost/tokenizer.hpp>
  33. const U32 KEYWORD_FILE_CURRENT_VERSION = 2;
  34. inline BOOL LLKeywordToken::isHead(const llwchar* s) const
  35. {
  36. // strncmp is much faster than string compare
  37. BOOL res = TRUE;
  38. const llwchar* t = mToken.c_str();
  39. S32 len = mToken.size();
  40. for (S32 i=0; i<len; i++)
  41. {
  42. if (s[i] != t[i])
  43. {
  44. res = FALSE;
  45. break;
  46. }
  47. }
  48. return res;
  49. }
  50. LLKeywords::LLKeywords() : mLoaded(FALSE)
  51. {
  52. }
  53. inline BOOL LLKeywordToken::isTail(const llwchar* s) const
  54. {
  55. BOOL res = TRUE;
  56. const llwchar* t = mDelimiter.c_str();
  57. S32 len = mDelimiter.size();
  58. for (S32 i=0; i<len; i++)
  59. {
  60. if (s[i] != t[i])
  61. {
  62. res = FALSE;
  63. break;
  64. }
  65. }
  66. return res;
  67. }
  68. LLKeywords::~LLKeywords()
  69. {
  70. std::for_each(mWordTokenMap.begin(), mWordTokenMap.end(), DeletePairedPointer());
  71. std::for_each(mLineTokenList.begin(), mLineTokenList.end(), DeletePointer());
  72. std::for_each(mDelimiterTokenList.begin(), mDelimiterTokenList.end(), DeletePointer());
  73. }
  74. BOOL LLKeywords::loadFromFile( const std::string& filename )
  75. {
  76. mLoaded = FALSE;
  77. ////////////////////////////////////////////////////////////
  78. // File header
  79. const S32 BUFFER_SIZE = 1024;
  80. char buffer[BUFFER_SIZE]; /* Flawfinder: ignore */
  81. llifstream file;
  82. file.open(filename); /* Flawfinder: ignore */
  83. if( file.fail() )
  84. {
  85. llinfos << "LLKeywords::loadFromFile() Unable to open file: " << filename << llendl;
  86. return mLoaded;
  87. }
  88. // Identifying string
  89. file >> buffer;
  90. if( strcmp( buffer, "llkeywords" ) )
  91. {
  92. llinfos << filename << " does not appear to be a keyword file" << llendl;
  93. return mLoaded;
  94. }
  95. // Check file version
  96. file >> buffer;
  97. U32 version_num;
  98. file >> version_num;
  99. if( strcmp(buffer, "version") || version_num != (U32)KEYWORD_FILE_CURRENT_VERSION )
  100. {
  101. llinfos << filename << " does not appear to be a version " << KEYWORD_FILE_CURRENT_VERSION << " keyword file" << llendl;
  102. return mLoaded;
  103. }
  104. // start of line (SOL)
  105. std::string SOL_COMMENT("#");
  106. std::string SOL_WORD("[word ");
  107. std::string SOL_LINE("[line ");
  108. std::string SOL_ONE_SIDED_DELIMITER("[one_sided_delimiter ");
  109. std::string SOL_TWO_SIDED_DELIMITER("[two_sided_delimiter ");
  110. std::string SOL_DOUBLE_QUOTATION_MARKS("[double_quotation_marks ");
  111. LLColor3 cur_color( 1, 0, 0 );
  112. LLKeywordToken::TOKEN_TYPE cur_type = LLKeywordToken::WORD;
  113. while (!file.eof())
  114. {
  115. buffer[0] = 0;
  116. file.getline( buffer, BUFFER_SIZE );
  117. std::string line(buffer);
  118. if( line.find(SOL_COMMENT) == 0 )
  119. {
  120. continue;
  121. }
  122. else if( line.find(SOL_WORD) == 0 )
  123. {
  124. cur_color = readColor( line.substr(SOL_WORD.size()) );
  125. cur_type = LLKeywordToken::WORD;
  126. continue;
  127. }
  128. else if( line.find(SOL_LINE) == 0 )
  129. {
  130. cur_color = readColor( line.substr(SOL_LINE.size()) );
  131. cur_type = LLKeywordToken::LINE;
  132. continue;
  133. }
  134. else if( line.find(SOL_TWO_SIDED_DELIMITER) == 0 )
  135. {
  136. cur_color = readColor( line.substr(SOL_TWO_SIDED_DELIMITER.size()) );
  137. cur_type = LLKeywordToken::TWO_SIDED_DELIMITER;
  138. continue;
  139. }
  140. else if( line.find(SOL_DOUBLE_QUOTATION_MARKS) == 0 )
  141. {
  142. cur_color = readColor( line.substr(SOL_DOUBLE_QUOTATION_MARKS.size()) );
  143. cur_type = LLKeywordToken::DOUBLE_QUOTATION_MARKS;
  144. continue;
  145. }
  146. else if( line.find(SOL_ONE_SIDED_DELIMITER) == 0 )
  147. {
  148. cur_color = readColor( line.substr(SOL_ONE_SIDED_DELIMITER.size()) );
  149. cur_type = LLKeywordToken::ONE_SIDED_DELIMITER;
  150. continue;
  151. }
  152. std::string token_buffer( line );
  153. LLStringUtil::trim(token_buffer);
  154. typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
  155. boost::char_separator<char> sep_word("", " \t");
  156. tokenizer word_tokens(token_buffer, sep_word);
  157. tokenizer::iterator token_word_iter = word_tokens.begin();
  158. if( !token_buffer.empty() && token_word_iter != word_tokens.end() )
  159. {
  160. // first word is the keyword or a left delimiter
  161. std::string keyword = (*token_word_iter);
  162. LLStringUtil::trim(keyword);
  163. // second word may be a right delimiter
  164. std::string delimiter;
  165. if (cur_type == LLKeywordToken::TWO_SIDED_DELIMITER)
  166. {
  167. while (delimiter.length() == 0 && ++token_word_iter != word_tokens.end())
  168. {
  169. delimiter = *token_word_iter;
  170. LLStringUtil::trim(delimiter);
  171. }
  172. }
  173. else if (cur_type == LLKeywordToken::DOUBLE_QUOTATION_MARKS)
  174. {
  175. // Closing delimiter is identical to the opening one.
  176. delimiter = keyword;
  177. }
  178. // following words are tooltip
  179. std::string tool_tip;
  180. while (++token_word_iter != word_tokens.end())
  181. {
  182. tool_tip += (*token_word_iter);
  183. }
  184. LLStringUtil::trim(tool_tip);
  185. if( !tool_tip.empty() )
  186. {
  187. // Replace : with \n for multi-line tool tips.
  188. LLStringUtil::replaceChar( tool_tip, ':', '\n' );
  189. addToken(cur_type, keyword, cur_color, tool_tip, delimiter );
  190. }
  191. else
  192. {
  193. addToken(cur_type, keyword, cur_color, LLStringUtil::null, delimiter );
  194. }
  195. }
  196. }
  197. file.close();
  198. mLoaded = TRUE;
  199. return mLoaded;
  200. }
  201. // Add the token as described
  202. void LLKeywords::addToken(LLKeywordToken::TOKEN_TYPE type,
  203. const std::string& key_in,
  204. const LLColor3& color,
  205. const std::string& tool_tip_in,
  206. const std::string& delimiter_in)
  207. {
  208. LLWString key = utf8str_to_wstring(key_in);
  209. LLWString tool_tip = utf8str_to_wstring(tool_tip_in);
  210. LLWString delimiter = utf8str_to_wstring(delimiter_in);
  211. switch(type)
  212. {
  213. case LLKeywordToken::WORD:
  214. mWordTokenMap[key] = new LLKeywordToken(type, color, key, tool_tip, LLWStringUtil::null);
  215. break;
  216. case LLKeywordToken::LINE:
  217. mLineTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, LLWStringUtil::null));
  218. break;
  219. case LLKeywordToken::TWO_SIDED_DELIMITER:
  220. case LLKeywordToken::DOUBLE_QUOTATION_MARKS:
  221. case LLKeywordToken::ONE_SIDED_DELIMITER:
  222. mDelimiterTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter));
  223. break;
  224. default:
  225. llassert(0);
  226. }
  227. }
  228. LLKeywords::WStringMapIndex::WStringMapIndex(const WStringMapIndex& other)
  229. {
  230. if(other.mOwner)
  231. {
  232. copyData(other.mData, other.mLength);
  233. }
  234. else
  235. {
  236. mOwner = false;
  237. mLength = other.mLength;
  238. mData = other.mData;
  239. }
  240. }
  241. LLKeywords::WStringMapIndex::WStringMapIndex(const LLWString& str)
  242. {
  243. copyData(str.data(), str.size());
  244. }
  245. LLKeywords::WStringMapIndex::WStringMapIndex(const llwchar *start, size_t length):
  246. mData(start), mLength(length), mOwner(false)
  247. {
  248. }
  249. LLKeywords::WStringMapIndex::~WStringMapIndex()
  250. {
  251. if(mOwner)
  252. delete[] mData;
  253. }
  254. void LLKeywords::WStringMapIndex::copyData(const llwchar *start, size_t length)
  255. {
  256. llwchar *data = new llwchar[length];
  257. memcpy((void*)data, (const void*)start, length * sizeof(llwchar));
  258. mOwner = true;
  259. mLength = length;
  260. mData = data;
  261. }
  262. bool LLKeywords::WStringMapIndex::operator<(const LLKeywords::WStringMapIndex &other) const
  263. {
  264. // NOTE: Since this is only used to organize a std::map, it doesn't matter if it uses correct collate order or not.
  265. // The comparison only needs to strictly order all possible strings, and be stable.
  266. bool result = false;
  267. const llwchar* self_iter = mData;
  268. const llwchar* self_end = mData + mLength;
  269. const llwchar* other_iter = other.mData;
  270. const llwchar* other_end = other.mData + other.mLength;
  271. while(true)
  272. {
  273. if(other_iter >= other_end)
  274. {
  275. // We've hit the end of other.
  276. // This covers two cases: other being shorter than self, or the strings being equal.
  277. // In either case, we want to return false.
  278. result = false;
  279. break;
  280. }
  281. else if(self_iter >= self_end)
  282. {
  283. // self is shorter than other.
  284. result = true;
  285. break;
  286. }
  287. else if(*self_iter != *other_iter)
  288. {
  289. // The current character differs. The strings are not equal.
  290. result = *self_iter < *other_iter;
  291. break;
  292. }
  293. self_iter++;
  294. other_iter++;
  295. }
  296. return result;
  297. }
  298. LLColor3 LLKeywords::readColor( const std::string& s )
  299. {
  300. F32 r, g, b;
  301. r = g = b = 0.0f;
  302. S32 values_read = sscanf(s.c_str(), "%f, %f, %f]", &r, &g, &b );
  303. if( values_read != 3 )
  304. {
  305. llinfos << " poorly formed color in keyword file" << llendl;
  306. }
  307. return LLColor3( r, g, b );
  308. }
  309. LLFastTimer::DeclareTimer FTM_SYNTAX_COLORING("Syntax Coloring");
  310. // Walk through a string, applying the rules specified by the keyword token list and
  311. // create a list of color segments.
  312. void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLWString& wtext, const LLColor4 &defaultColor, LLTextEditor& editor)
  313. {
  314. LLFastTimer ft(FTM_SYNTAX_COLORING);
  315. seg_list->clear();
  316. if( wtext.empty() )
  317. {
  318. return;
  319. }
  320. S32 text_len = wtext.size() + 1;
  321. seg_list->push_back( new LLNormalTextSegment( defaultColor, 0, text_len, editor ) );
  322. const llwchar* base = wtext.c_str();
  323. const llwchar* cur = base;
  324. const llwchar* line = NULL;
  325. while( *cur )
  326. {
  327. if( *cur == '\n' || cur == base )
  328. {
  329. if( *cur == '\n' )
  330. {
  331. LLTextSegmentPtr text_segment = new LLLineBreakTextSegment(cur-base);
  332. text_segment->setToken( 0 );
  333. insertSegment( *seg_list, text_segment, text_len, defaultColor, editor);
  334. cur++;
  335. if( !*cur || *cur == '\n' )
  336. {
  337. continue;
  338. }
  339. }
  340. // Start of a new line
  341. line = cur;
  342. // Skip white space
  343. while( *cur && isspace(*cur) && (*cur != '\n') )
  344. {
  345. cur++;
  346. }
  347. if( !*cur || *cur == '\n' )
  348. {
  349. continue;
  350. }
  351. // cur is now at the first non-whitespace character of a new line
  352. // Line start tokens
  353. {
  354. BOOL line_done = FALSE;
  355. for (token_list_t::iterator iter = mLineTokenList.begin();
  356. iter != mLineTokenList.end(); ++iter)
  357. {
  358. LLKeywordToken* cur_token = *iter;
  359. if( cur_token->isHead( cur ) )
  360. {
  361. S32 seg_start = cur - base;
  362. while( *cur && *cur != '\n' )
  363. {
  364. // skip the rest of the line
  365. cur++;
  366. }
  367. S32 seg_end = cur - base;
  368. //create segments from seg_start to seg_end
  369. insertSegments(wtext, *seg_list,cur_token, text_len, seg_start, seg_end, defaultColor, editor);
  370. line_done = TRUE; // to break out of second loop.
  371. break;
  372. }
  373. }
  374. if( line_done )
  375. {
  376. continue;
  377. }
  378. }
  379. }
  380. // Skip white space
  381. while( *cur && isspace(*cur) && (*cur != '\n') )
  382. {
  383. cur++;
  384. }
  385. while( *cur && *cur != '\n' )
  386. {
  387. // Check against delimiters
  388. {
  389. S32 seg_start = 0;
  390. LLKeywordToken* cur_delimiter = NULL;
  391. for (token_list_t::iterator iter = mDelimiterTokenList.begin();
  392. iter != mDelimiterTokenList.end(); ++iter)
  393. {
  394. LLKeywordToken* delimiter = *iter;
  395. if( delimiter->isHead( cur ) )
  396. {
  397. cur_delimiter = delimiter;
  398. break;
  399. }
  400. }
  401. if( cur_delimiter )
  402. {
  403. S32 between_delimiters = 0;
  404. S32 seg_end = 0;
  405. seg_start = cur - base;
  406. cur += cur_delimiter->getLengthHead();
  407. LLKeywordToken::TOKEN_TYPE type = cur_delimiter->getType();
  408. if( type == LLKeywordToken::TWO_SIDED_DELIMITER || type == LLKeywordToken::DOUBLE_QUOTATION_MARKS )
  409. {
  410. while( *cur && !cur_delimiter->isTail(cur))
  411. {
  412. // Check for an escape sequence.
  413. if (type == LLKeywordToken::DOUBLE_QUOTATION_MARKS && *cur == '\\')
  414. {
  415. // Count the number of backslashes.
  416. S32 num_backslashes = 0;
  417. while (*cur == '\\')
  418. {
  419. num_backslashes++;
  420. between_delimiters++;
  421. cur++;
  422. }
  423. // If the next character is the end delimiter?
  424. if (cur_delimiter->isTail(cur))
  425. {
  426. // If there was an odd number of backslashes, then this delimiter
  427. // does not end the sequence.
  428. if (num_backslashes % 2 == 1)
  429. {
  430. between_delimiters++;
  431. cur++;
  432. }
  433. else
  434. {
  435. // This is an end delimiter.
  436. break;
  437. }
  438. }
  439. }
  440. else
  441. {
  442. between_delimiters++;
  443. cur++;
  444. }
  445. }
  446. if( *cur )
  447. {
  448. cur += cur_delimiter->getLengthHead();
  449. seg_end = seg_start + between_delimiters + cur_delimiter->getLengthHead() + cur_delimiter->getLengthTail();
  450. }
  451. else
  452. {
  453. // eof
  454. seg_end = seg_start + between_delimiters + cur_delimiter->getLengthHead();
  455. }
  456. }
  457. else
  458. {
  459. llassert( cur_delimiter->getType() == LLKeywordToken::ONE_SIDED_DELIMITER );
  460. // Left side is the delimiter. Right side is eol or eof.
  461. while( *cur && ('\n' != *cur) )
  462. {
  463. between_delimiters++;
  464. cur++;
  465. }
  466. seg_end = seg_start + between_delimiters + cur_delimiter->getLengthHead();
  467. }
  468. insertSegments(wtext, *seg_list,cur_delimiter, text_len, seg_start, seg_end, defaultColor, editor);
  469. /*
  470. LLTextSegmentPtr text_segment = new LLNormalTextSegment( cur_delimiter->getColor(), seg_start, seg_end, editor );
  471. text_segment->setToken( cur_delimiter );
  472. insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
  473. */
  474. // Note: we don't increment cur, since the end of one delimited seg may be immediately
  475. // followed by the start of another one.
  476. continue;
  477. }
  478. }
  479. // check against words
  480. llwchar prev = cur > base ? *(cur-1) : 0;
  481. if( !isalnum( prev ) && (prev != '_') )
  482. {
  483. const llwchar* p = cur;
  484. while( isalnum( *p ) || (*p == '_') )
  485. {
  486. p++;
  487. }
  488. S32 seg_len = p - cur;
  489. if( seg_len > 0 )
  490. {
  491. WStringMapIndex word( cur, seg_len );
  492. word_token_map_t::iterator map_iter = mWordTokenMap.find(word);
  493. if( map_iter != mWordTokenMap.end() )
  494. {
  495. LLKeywordToken* cur_token = map_iter->second;
  496. S32 seg_start = cur - base;
  497. S32 seg_end = seg_start + seg_len;
  498. // llinfos << "Seg: [" << word.c_str() << "]" << llendl;
  499. insertSegments(wtext, *seg_list,cur_token, text_len, seg_start, seg_end, defaultColor, editor);
  500. }
  501. cur += seg_len;
  502. continue;
  503. }
  504. }
  505. if( *cur && *cur != '\n' )
  506. {
  507. cur++;
  508. }
  509. }
  510. }
  511. }
  512. void LLKeywords::insertSegments(const LLWString& wtext, std::vector<LLTextSegmentPtr>& seg_list, LLKeywordToken* cur_token, S32 text_len, S32 seg_start, S32 seg_end, const LLColor4 &defaultColor, LLTextEditor& editor )
  513. {
  514. std::string::size_type pos = wtext.find('\n',seg_start);
  515. while (pos!=-1 && pos < (std::string::size_type)seg_end)
  516. {
  517. if (pos!=seg_start)
  518. {
  519. LLTextSegmentPtr text_segment = new LLNormalTextSegment( cur_token->getColor(), seg_start, pos, editor );
  520. text_segment->setToken( cur_token );
  521. insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
  522. }
  523. LLTextSegmentPtr text_segment = new LLLineBreakTextSegment(pos);
  524. text_segment->setToken( cur_token );
  525. insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
  526. seg_start = pos+1;
  527. pos = wtext.find('\n',seg_start);
  528. }
  529. LLTextSegmentPtr text_segment = new LLNormalTextSegment( cur_token->getColor(), seg_start, seg_end, editor );
  530. text_segment->setToken( cur_token );
  531. insertSegment( seg_list, text_segment, text_len, defaultColor, editor);
  532. }
  533. void LLKeywords::insertSegment(std::vector<LLTextSegmentPtr>& seg_list, LLTextSegmentPtr new_segment, S32 text_len, const LLColor4 &defaultColor, LLTextEditor& editor )
  534. {
  535. LLTextSegmentPtr last = seg_list.back();
  536. S32 new_seg_end = new_segment->getEnd();
  537. if( new_segment->getStart() == last->getStart() )
  538. {
  539. seg_list.pop_back();
  540. }
  541. else
  542. {
  543. last->setEnd( new_segment->getStart() );
  544. }
  545. seg_list.push_back( new_segment );
  546. if( new_seg_end < text_len )
  547. {
  548. seg_list.push_back( new LLNormalTextSegment( defaultColor, new_seg_end, text_len, editor ) );
  549. }
  550. }
  551. #ifdef _DEBUG
  552. void LLKeywords::dump()
  553. {
  554. llinfos << "LLKeywords" << llendl;
  555. llinfos << "LLKeywords::sWordTokenMap" << llendl;
  556. word_token_map_t::iterator word_token_iter = mWordTokenMap.begin();
  557. while( word_token_iter != mWordTokenMap.end() )
  558. {
  559. LLKeywordToken* word_token = word_token_iter->second;
  560. word_token->dump();
  561. ++word_token_iter;
  562. }
  563. llinfos << "LLKeywords::sLineTokenList" << llendl;
  564. for (token_list_t::iterator iter = mLineTokenList.begin();
  565. iter != mLineTokenList.end(); ++iter)
  566. {
  567. LLKeywordToken* line_token = *iter;
  568. line_token->dump();
  569. }
  570. llinfos << "LLKeywords::sDelimiterTokenList" << llendl;
  571. for (token_list_t::iterator iter = mDelimiterTokenList.begin();
  572. iter != mDelimiterTokenList.end(); ++iter)
  573. {
  574. LLKeywordToken* delimiter_token = *iter;
  575. delimiter_token->dump();
  576. }
  577. }
  578. void LLKeywordToken::dump()
  579. {
  580. llinfos << "[" <<
  581. mColor.mV[VX] << ", " <<
  582. mColor.mV[VY] << ", " <<
  583. mColor.mV[VZ] << "] [" <<
  584. wstring_to_utf8str(mToken) << "]" <<
  585. llendl;
  586. }
  587. #endif // DEBUG