/native/external/espeak/src/compiledict.cpp

http://eyes-free.googlecode.com/ · C++ · 1556 lines · 1266 code · 224 blank · 66 comment · 270 complexity · a850cefc9354c5188dc75d1180d217d4 MD5 · raw file

  1. /***************************************************************************
  2. * Copyright (C) 2005 to 2007 by Jonathan Duddington *
  3. * email: jonsd@users.sourceforge.net *
  4. * *
  5. * This program is free software; you can redistribute it and/or modify *
  6. * it under the terms of the GNU General Public License as published by *
  7. * the Free Software Foundation; either version 3 of the License, or *
  8. * (at your option) any later version. *
  9. * *
  10. * This program is distributed in the hope that it will be useful, *
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of *
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
  13. * GNU General Public License for more details. *
  14. * *
  15. * You should have received a copy of the GNU General Public License *
  16. * along with this program; if not, write see: *
  17. * <http://www.gnu.org/licenses/>. *
  18. ***************************************************************************/
  19. #include "StdAfx.h"
  20. #include <stdio.h>
  21. #include <ctype.h>
  22. #include <stdlib.h>
  23. #include <string.h>
  24. #include <wctype.h>
  25. #include "speak_lib.h"
  26. #include "speech.h"
  27. #include "phoneme.h"
  28. #include "synthesize.h"
  29. #include "translate.h"
  30. //#define OPT_FORMAT // format the text and write formatted copy to Log file
  31. //#define OUTPUT_FORMAT
  32. extern void Write4Bytes(FILE *f, int value);
  33. int HashDictionary(const char *string);
  34. static FILE *f_log = NULL;
  35. extern char *dir_dictionary;
  36. int linenum;
  37. static int error_count;
  38. static int transpose_offset; // transpose character range for LookupDictList()
  39. static int transpose_min;
  40. static int transpose_max;
  41. static int text_mode = 0;
  42. static int debug_flag = 0;
  43. int hash_counts[N_HASH_DICT];
  44. char *hash_chains[N_HASH_DICT];
  45. MNEM_TAB mnem_flags[] = {
  46. // these in the first group put a value in bits0-3 of dictionary_flags
  47. {"$1", 0x41}, // stress on 1st syllable
  48. {"$2", 0x42}, // stress on 2nd syllable
  49. {"$3", 0x43},
  50. {"$4", 0x44},
  51. {"$5", 0x45},
  52. {"$6", 0x46},
  53. {"$7", 0x47},
  54. {"$u", 0x48}, // reduce to unstressed
  55. {"$u1", 0x49},
  56. {"$u2", 0x4a},
  57. {"$u3", 0x4b},
  58. {"$u+", 0x4c}, // reduce to unstressed, but stress at end of clause
  59. {"$u1+", 0x4d},
  60. {"$u2+", 0x4e},
  61. {"$u3+", 0x4f},
  62. // these set the corresponding numbered bit if dictionary_flags
  63. {"$pause", 8}, /* ensure pause before this word */
  64. {"$only", 9}, /* only match on this word without suffix */
  65. {"$onlys", 10}, /* only match with none, or with 's' suffix */
  66. {"$strend", 11}, /* full stress if at end of clause */
  67. {"$strend2", 12}, /* full stress if at end of clause, or only followed by unstressed */
  68. {"$unstressend",13}, /* reduce stress at end of clause */
  69. {"$atend", 14}, /* use this pronunciation if at end of clause */
  70. {"$dot", 16}, /* ignore '.' after this word (abbreviation) */
  71. {"$abbrev", 17}, /* use this pronuciation rather than split into letters */
  72. {"$stem", 18}, // must have a suffix
  73. // language specific
  74. {"$double", 19}, // IT double the initial consonant of next word
  75. {"$alt", 20}, // use alternative pronunciation
  76. {"$alt2", 21},
  77. {"$brk", 28}, // a shorter $pause
  78. {"$text", 29}, // word translates to replcement text, not phonemes
  79. // flags in dictionary word 2
  80. {"$verbf", 0x20}, /* verb follows */
  81. {"$verbsf", 0x21}, /* verb follows, allow -s suffix */
  82. {"$nounf", 0x22}, /* noun follows */
  83. {"$pastf", 0x23}, /* past tense follows */
  84. {"$verb", 0x24}, /* use this pronunciation when its a verb */
  85. {"$noun", 0x25}, /* use this pronunciation when its a noun */
  86. {"$past", 0x26}, /* use this pronunciation when its past tense */
  87. {"$verbextend",0x28}, /* extend influence of 'verb follows' */
  88. {"$capital", 0x29}, /* use this pronunciation if initial letter is upper case */
  89. {"$allcaps", 0x2a}, /* use this pronunciation if initial letter is upper case */
  90. {"$accent", 0x2b}, // character name is base-character name + accent name
  91. // doesn't set dictionary_flags
  92. {"$?", 100}, // conditional rule, followed by byte giving the condition number
  93. {"$textmode", 200},
  94. {"$phonememode", 201},
  95. {NULL, -1}
  96. };
  97. #define LEN_GROUP_NAME 12
  98. typedef struct {
  99. char name[LEN_GROUP_NAME+1];
  100. unsigned int start;
  101. unsigned int length;
  102. } RGROUP;
  103. int isspace2(unsigned int c)
  104. {//=========================
  105. // can't use isspace() because on Windows, isspace(0xe1) gives TRUE !
  106. int c2;
  107. if(((c2 = (c & 0xff)) == 0) || (c > ' '))
  108. return(0);
  109. return(1);
  110. }
  111. static FILE *fopen_log(const char *fname,const char *access)
  112. {//==================================================
  113. // performs fopen, but produces error message to f_log if it fails
  114. FILE *f;
  115. if((f = fopen(fname,access)) == NULL)
  116. {
  117. if(f_log != NULL)
  118. fprintf(f_log,"Can't access (%s) file '%s'\n",access,fname);
  119. }
  120. return(f);
  121. }
  122. #ifdef OPT_FORMAT
  123. static const char *lookup_mnem(MNEM_TAB *table, int value)
  124. //========================================================
  125. /* Lookup a mnemonic string in a table, return its name */
  126. {
  127. while(table->mnem != NULL)
  128. {
  129. if(table->value==value)
  130. return(table->mnem);
  131. table++;
  132. }
  133. return("??"); /* not found */
  134. } /* end of mnem */
  135. #endif
  136. int compile_line(char *linebuf, char *dict_line, int *hash)
  137. {//========================================================
  138. // Compile a line in the language_list file
  139. unsigned char c;
  140. char *p;
  141. char *word;
  142. char *phonetic;
  143. unsigned int ix;
  144. int step;
  145. unsigned int n_flag_codes = 0;
  146. int flag_offset;
  147. int length;
  148. int multiple_words = 0;
  149. char *multiple_string = NULL;
  150. char *multiple_string_end = NULL;
  151. int len_word;
  152. int len_phonetic;
  153. int text_not_phonemes; // this word specifies replacement text, not phonemes
  154. unsigned int wc;
  155. char *mnemptr;
  156. char *comment;
  157. unsigned char flag_codes[100];
  158. char encoded_ph[200];
  159. unsigned char bad_phoneme[4];
  160. static char nullstring[] = {0};
  161. comment = NULL;
  162. text_not_phonemes = 0;
  163. phonetic = word = nullstring;
  164. p = linebuf;
  165. // while(isspace2(*p)) p++;
  166. #ifdef deleted
  167. if(*p == '$')
  168. {
  169. if(memcmp(p,"$textmode",9) == 0)
  170. {
  171. text_mode = 1;
  172. return(0);
  173. }
  174. if(memcmp(p,"$phonememode",12) == 0)
  175. {
  176. text_mode = 0;
  177. return(0);
  178. }
  179. }
  180. #endif
  181. step = 0;
  182. c = 0;
  183. while(c != '\n')
  184. {
  185. c = *p;
  186. if((c == '?') && (step==0))
  187. {
  188. // conditional rule, allow only if the numbered condition is set for the voice
  189. flag_offset = 100;
  190. p++;
  191. if(*p == '!')
  192. {
  193. // allow only if the numbered condition is NOT set
  194. flag_offset = 132;
  195. p++;
  196. }
  197. ix = 0;
  198. if(isdigit(*p))
  199. {
  200. ix += (*p-'0');
  201. p++;
  202. }
  203. if(isdigit(*p))
  204. {
  205. ix = ix*10 + (*p-'0');
  206. p++;
  207. }
  208. flag_codes[n_flag_codes++] = ix + flag_offset;
  209. c = *p;
  210. }
  211. if((c == '$') && isalnum(p[1]))
  212. {
  213. /* read keyword parameter */
  214. mnemptr = p;
  215. while(!isspace2(c = *p)) p++;
  216. *p = 0;
  217. ix = LookupMnem(mnem_flags,mnemptr);
  218. if(ix > 0)
  219. {
  220. if(ix == 200)
  221. {
  222. text_mode = 1;
  223. }
  224. else
  225. if(ix == 201)
  226. {
  227. text_mode = 0;
  228. }
  229. else
  230. if(ix == BITNUM_FLAG_TEXTMODE)
  231. {
  232. text_not_phonemes = 1;
  233. }
  234. else
  235. {
  236. flag_codes[n_flag_codes++] = ix;
  237. }
  238. }
  239. else
  240. {
  241. fprintf(f_log,"%5d: Unknown keyword: %s\n",linenum,mnemptr);
  242. error_count++;
  243. }
  244. }
  245. if((c == '/') && (p[1] == '/') && (multiple_words==0))
  246. {
  247. c = '\n'; /* "//" treat comment as end of line */
  248. comment = p;
  249. }
  250. switch(step)
  251. {
  252. case 0:
  253. if(c == '(')
  254. {
  255. multiple_words = 1;
  256. word = p+1;
  257. step = 1;
  258. }
  259. else
  260. if(!isspace2(c))
  261. {
  262. word = p;
  263. step = 1;
  264. }
  265. break;
  266. case 1:
  267. if(isspace2(c))
  268. {
  269. p[0] = 0; /* terminate english word */
  270. if(multiple_words)
  271. {
  272. multiple_string = multiple_string_end = p+1;
  273. step = 2;
  274. }
  275. else
  276. {
  277. step = 3;
  278. }
  279. }
  280. else
  281. if((c == ')') && multiple_words)
  282. {
  283. p[0] = 0;
  284. step = 3;
  285. multiple_words = 0;
  286. }
  287. break;
  288. case 2:
  289. if(isspace2(c))
  290. {
  291. multiple_words++;
  292. }
  293. else
  294. if(c == ')')
  295. {
  296. p[0] = ' '; // terminate extra string
  297. multiple_string_end = p+1;
  298. step = 3;
  299. }
  300. break;
  301. case 3:
  302. if(!isspace2(c))
  303. {
  304. phonetic = p;
  305. step = 4;
  306. }
  307. break;
  308. case 4:
  309. if(isspace2(c))
  310. {
  311. p[0] = 0; /* terminate phonetic */
  312. step = 5;
  313. }
  314. break;
  315. case 5:
  316. break;
  317. }
  318. p++;
  319. }
  320. if(word[0] == 0)
  321. {
  322. #ifdef OPT_FORMAT
  323. if(comment != NULL)
  324. fprintf(f_log,"%s",comment);
  325. else
  326. fputc('\n',f_log);
  327. #endif
  328. return(0); /* blank line */
  329. }
  330. if(text_mode)
  331. text_not_phonemes = 1;
  332. if(text_not_phonemes != translator->langopts.textmode)
  333. {
  334. flag_codes[n_flag_codes++] = BITNUM_FLAG_TEXTMODE;
  335. }
  336. if(text_not_phonemes)
  337. {
  338. // this is replacement text, so don't encode as phonemes. Restrict the length of the replacement word
  339. strncpy0(encoded_ph,phonetic,N_WORD_BYTES-4);
  340. }
  341. else
  342. {
  343. EncodePhonemes(phonetic,encoded_ph,bad_phoneme);
  344. if(strchr(encoded_ph,phonSWITCH) != 0)
  345. {
  346. flag_codes[n_flag_codes++] = BITNUM_FLAG_ONLY_S; // don't match on suffixes (except 's') when switching languages
  347. }
  348. // check for errors in the phonemes codes
  349. for(ix=0; ix<sizeof(encoded_ph); ix++)
  350. {
  351. c = encoded_ph[ix];
  352. if(c == 0) break;
  353. if(c == 255)
  354. {
  355. /* unrecognised phoneme, report error */
  356. fprintf(f_log,"%5d: Bad phoneme [%c] (0x%x) in: %s %s\n",linenum,bad_phoneme[0],bad_phoneme[0],word,phonetic);
  357. error_count++;
  358. }
  359. }
  360. }
  361. if(sscanf(word,"U+%x",&wc) == 1)
  362. {
  363. // Character code
  364. ix = utf8_out(wc, word);
  365. word[ix] = 0;
  366. }
  367. else
  368. if((word[0] & 0x80)==0) // 7 bit ascii only
  369. {
  370. // If first letter is uppercase, convert to lower case. (Only if it's 7bit ascii)
  371. // ??? need to consider utf8 here
  372. word[0] = tolower(word[0]);
  373. }
  374. len_word = strlen(word);
  375. if(transpose_offset > 0)
  376. {
  377. len_word = TransposeAlphabet(word, transpose_offset, transpose_min, transpose_max);
  378. }
  379. *hash = HashDictionary(word);
  380. len_phonetic = strlen(encoded_ph);
  381. dict_line[1] = len_word; // bit 6 indicates whether the word has been compressed
  382. len_word &= 0x3f;
  383. memcpy(&dict_line[2],word,len_word);
  384. if(len_phonetic == 0)
  385. {
  386. // no phonemes specified. set bit 7
  387. dict_line[1] |= 0x80;
  388. length = len_word + 2;
  389. }
  390. else
  391. {
  392. length = len_word + len_phonetic + 3;
  393. strcpy(&dict_line[(len_word)+2],encoded_ph);
  394. }
  395. for(ix=0; ix<n_flag_codes; ix++)
  396. {
  397. dict_line[ix+length] = flag_codes[ix];
  398. }
  399. length += n_flag_codes;
  400. if((multiple_string != NULL) && (multiple_words > 0))
  401. {
  402. if(multiple_words > 10)
  403. {
  404. fprintf(f_log,"%5d: Two many parts in a multi-word entry: %d\n",linenum,multiple_words);
  405. }
  406. else
  407. {
  408. dict_line[length++] = 80 + multiple_words;
  409. ix = multiple_string_end - multiple_string;
  410. memcpy(&dict_line[length],multiple_string,ix);
  411. length += ix;
  412. }
  413. }
  414. dict_line[0] = length;
  415. #ifdef OPT_FORMAT
  416. spaces = 16;
  417. for(ix=0; ix<n_flag_codes; ix++)
  418. {
  419. if(flag_codes[ix] >= 100)
  420. {
  421. fprintf(f_log,"?%d ",flag_codes[ix]-100);
  422. spaces -= 3;
  423. }
  424. }
  425. fprintf(f_log,"%s",word);
  426. spaces -= strlen(word);
  427. DecodePhonemes(encoded_ph,decoded_ph);
  428. while(spaces-- > 0) fputc(' ',f_log);
  429. spaces += (14 - strlen(decoded_ph));
  430. fprintf(f_log," %s",decoded_ph);
  431. while(spaces-- > 0) fputc(' ',f_log);
  432. for(ix=0; ix<n_flag_codes; ix++)
  433. {
  434. if(flag_codes[ix] < 100)
  435. fprintf(f_log," %s",lookup_mnem(mnem_flags,flag_codes[ix]));
  436. }
  437. if(comment != NULL)
  438. fprintf(f_log," %s",comment);
  439. else
  440. fputc('\n',f_log);
  441. #endif
  442. return(length);
  443. } /* end of compile_line */
  444. void compile_dictlist_start(void)
  445. {//==============================
  446. // initialise dictionary list
  447. int ix;
  448. char *p;
  449. char *p2;
  450. for(ix=0; ix<N_HASH_DICT; ix++)
  451. {
  452. p = hash_chains[ix];
  453. while(p != NULL)
  454. {
  455. memcpy(&p2,p,sizeof(char *));
  456. free(p);
  457. p = p2;
  458. }
  459. hash_chains[ix] = NULL;
  460. hash_counts[ix]=0;
  461. }
  462. }
  463. void compile_dictlist_end(FILE *f_out)
  464. {//===================================
  465. // Write out the compiled dictionary list
  466. int hash;
  467. int length;
  468. char *p;
  469. if(f_log != NULL)
  470. {
  471. #ifdef OUTPUT_FORMAT
  472. for(hash=0; hash<N_HASH_DICT; hash++)
  473. {
  474. fprintf(f_log,"%8d",hash_counts[hash]);
  475. if((hash & 7) == 7)
  476. fputc('\n',f_log);
  477. }
  478. fflush(f_log);
  479. #endif
  480. }
  481. for(hash=0; hash<N_HASH_DICT; hash++)
  482. {
  483. p = hash_chains[hash];
  484. hash_counts[hash] = (int)ftell(f_out);
  485. while(p != NULL)
  486. {
  487. length = *(p+sizeof(char *));
  488. fwrite(p+sizeof(char *),length,1,f_out);
  489. memcpy(&p,p,sizeof(char *));
  490. }
  491. fputc(0,f_out);
  492. }
  493. }
  494. int compile_dictlist_file(const char *path, const char* filename)
  495. {//==============================================================
  496. int length;
  497. int hash;
  498. char *p;
  499. int count=0;
  500. FILE *f_in;
  501. char buf[200];
  502. char fname[sizeof(path_home)+45];
  503. char dict_line[128];
  504. text_mode = 0;
  505. sprintf(fname,"%s%s",path,filename);
  506. if((f_in = fopen(fname,"r")) == NULL)
  507. return(-1);
  508. fprintf(f_log,"Compiling: '%s'\n",fname);
  509. linenum=0;
  510. while(fgets(buf,sizeof(buf),f_in) != NULL)
  511. {
  512. linenum++;
  513. length = compile_line(buf,dict_line,&hash);
  514. if(length == 0) continue; /* blank line */
  515. hash_counts[hash]++;
  516. p = (char *)malloc(length+sizeof(char *));
  517. if(p == NULL)
  518. {
  519. if(f_log != NULL)
  520. {
  521. fprintf(f_log,"Can't allocate memory\n");
  522. error_count++;
  523. }
  524. break;
  525. }
  526. memcpy(p,&hash_chains[hash],sizeof(char *));
  527. hash_chains[hash] = p;
  528. memcpy(p+sizeof(char *),dict_line,length);
  529. count++;
  530. }
  531. fprintf(f_log,"\t%d entries\n",count);
  532. fclose(f_in);
  533. return(0);
  534. } /* end of compile_dictlist_file */
  535. char rule_cond[80];
  536. char rule_pre[80];
  537. char rule_post[80];
  538. char rule_match[80];
  539. char rule_phonemes[80];
  540. char group_name[LEN_GROUP_NAME+1];
  541. #define N_RULES 2000 // max rules for each group
  542. int hexdigit(char c)
  543. {//=================
  544. if(isdigit(c))
  545. return(c - '0');
  546. return(tolower(c) - 'a' + 10);
  547. }
  548. void copy_rule_string(char *string, int &state)
  549. {//============================================
  550. // state 0: conditional, 1=pre, 2=match, 3=post, 4=phonemes
  551. static char *outbuf[5] = {rule_cond, rule_pre, rule_match, rule_post, rule_phonemes};
  552. static int next_state[5] = {2,2,4,4,4};
  553. char *output;
  554. char *p;
  555. int ix;
  556. int len;
  557. char c;
  558. int sxflags;
  559. int value;
  560. int literal;
  561. if(string[0] == 0) return;
  562. output = outbuf[state];
  563. if(state==4)
  564. {
  565. // append to any previous phoneme string, i.e. allow spaces in the phoneme string
  566. len = strlen(rule_phonemes);
  567. if(len > 0)
  568. rule_phonemes[len++] = ' ';
  569. output = &rule_phonemes[len];
  570. }
  571. sxflags = 0x808000; // to ensure non-zero bytes
  572. for(p=string,ix=0;;)
  573. {
  574. literal = 0;
  575. c = *p++;
  576. if(c == '\\')
  577. {
  578. c = *p++; // treat next character literally
  579. if((c >= '0') && (c <= '3') && (p[0] >= '0') && (p[0] <= '7') && (p[1] >= '0') && (p[1] <= '7'))
  580. {
  581. // character code given by 3 digit octal value;
  582. c = (c-'0')*64 + (p[0]-'0')*8 + (p[1]-'0');
  583. p += 2;
  584. }
  585. literal = 1;
  586. }
  587. if((state==1) || (state==3))
  588. {
  589. // replace special characters (note: 'E' is reserved for a replaced silent 'e')
  590. if(literal == 0)
  591. {
  592. static const char lettergp_letters[9] = {LETTERGP_A,LETTERGP_B,LETTERGP_C,0,0,LETTERGP_F,LETTERGP_G,LETTERGP_H,LETTERGP_Y};
  593. switch(c)
  594. {
  595. case '_':
  596. c = RULE_SPACE;
  597. break;
  598. case 'Y':
  599. c = 'I'; // drop through to next case
  600. case 'A': // vowel
  601. case 'B':
  602. case 'C':
  603. case 'H':
  604. case 'F':
  605. case 'G':
  606. if(state == 1)
  607. {
  608. // pre-rule, put the number before the RULE_LETTERGP;
  609. output[ix++] = lettergp_letters[c-'A'] + 'A';
  610. c = RULE_LETTERGP;
  611. }
  612. else
  613. {
  614. output[ix++] = RULE_LETTERGP;
  615. c = lettergp_letters[c-'A'] + 'A';
  616. }
  617. break;
  618. case 'D':
  619. c = RULE_DIGIT;
  620. break;
  621. case 'K':
  622. c = RULE_NOTVOWEL;
  623. break;
  624. case 'N':
  625. c = RULE_NO_SUFFIX;
  626. break;
  627. case 'V':
  628. c = RULE_IFVERB;
  629. break;
  630. case 'Z':
  631. c = RULE_NONALPHA;
  632. break;
  633. case '+':
  634. c = RULE_INC_SCORE;
  635. break;
  636. case '@':
  637. c = RULE_SYLLABLE;
  638. break;
  639. case '&':
  640. c = RULE_STRESSED;
  641. break;
  642. case '%':
  643. c = RULE_DOUBLE;
  644. break;
  645. case '#':
  646. c = RULE_DEL_FWD;
  647. break;
  648. case '!':
  649. c = RULE_CAPITAL;
  650. break;
  651. case 'T':
  652. c = RULE_ALT1;
  653. break;
  654. case 'W':
  655. c = RULE_SPELLING;
  656. break;
  657. case 'X':
  658. c = RULE_NOVOWELS;
  659. break;
  660. case 'L':
  661. // expect two digits
  662. c = *p++ - '0';
  663. value = *p++ - '0';
  664. c = c * 10 + value;
  665. if((value < 0) || (value > 9) || (c <= 0) || (c >= N_LETTER_GROUPS))
  666. {
  667. c = 0;
  668. fprintf(f_log,"%5d: Expected 2 digits after 'L'",linenum);
  669. error_count++;
  670. }
  671. c += 'A';
  672. if(state == 1)
  673. {
  674. // pre-rule, put the group number before the RULE_LETTERGP command
  675. output[ix++] = c;
  676. c = RULE_LETTERGP2;
  677. }
  678. else
  679. {
  680. output[ix++] = RULE_LETTERGP2;
  681. }
  682. break;
  683. case 'P':
  684. sxflags |= SUFX_P; // Prefix, now drop through to Suffix
  685. case '$': // obsolete, replaced by S
  686. case 'S':
  687. output[ix++] = RULE_ENDING;
  688. value = 0;
  689. while(!isspace2(c = *p++) && (c != 0))
  690. {
  691. switch(c)
  692. {
  693. case 'e':
  694. sxflags |= SUFX_E;
  695. break;
  696. case 'i':
  697. sxflags |= SUFX_I;
  698. break;
  699. case 'p': // obsolete, replaced by 'P' above
  700. sxflags |= SUFX_P;
  701. break;
  702. case 'v':
  703. sxflags |= SUFX_V;
  704. break;
  705. case 'd':
  706. sxflags |= SUFX_D;
  707. break;
  708. case 'f':
  709. sxflags |= SUFX_F;
  710. break;
  711. case 'q':
  712. sxflags |= SUFX_Q;
  713. break;
  714. case 't':
  715. sxflags |= SUFX_T;
  716. break;
  717. case 'b':
  718. sxflags |= SUFX_B;
  719. break;
  720. default:
  721. if(isdigit(c))
  722. value = (value*10) + (c - '0');
  723. break;
  724. }
  725. }
  726. p--;
  727. output[ix++] = sxflags >> 16;
  728. output[ix++] = sxflags >> 8;
  729. c = value | 0x80;
  730. break;
  731. }
  732. }
  733. }
  734. output[ix++] = c;
  735. if(c == 0) break;
  736. }
  737. state = next_state[state];
  738. } // end of copy_rule_string
  739. char *compile_rule(char *input)
  740. {//============================
  741. int ix;
  742. unsigned char c;
  743. int wc;
  744. char *p;
  745. char *prule;
  746. int len;
  747. int len_name;
  748. int state=2;
  749. int finish=0;
  750. int pre_bracket=0;
  751. char buf[80];
  752. char output[150];
  753. unsigned char bad_phoneme[4];
  754. buf[0]=0;
  755. rule_cond[0]=0;
  756. rule_pre[0]=0;
  757. rule_post[0]=0;
  758. rule_match[0]=0;
  759. rule_phonemes[0]=0;
  760. p = buf;
  761. for(ix=0; finish==0; ix++)
  762. {
  763. c = input[ix];
  764. switch(c = input[ix])
  765. {
  766. case ')': // end of prefix section
  767. *p = 0;
  768. state = 1;
  769. pre_bracket = 1;
  770. copy_rule_string(buf,state);
  771. p = buf;
  772. break;
  773. case '(': // start of suffix section
  774. *p = 0;
  775. state = 2;
  776. copy_rule_string(buf,state);
  777. state = 3;
  778. p = buf;
  779. break;
  780. case '\n': // end of line
  781. case '\r':
  782. case 0: // end of line
  783. *p = 0;
  784. copy_rule_string(buf,state);
  785. finish=1;
  786. break;
  787. case '\t': // end of section section
  788. case ' ':
  789. *p = 0;
  790. copy_rule_string(buf,state);
  791. p = buf;
  792. break;
  793. case '?':
  794. if(state==2)
  795. state=0;
  796. else
  797. *p++ = c;
  798. break;
  799. default:
  800. *p++ = c;
  801. break;
  802. }
  803. }
  804. if(strcmp(rule_match,"$group")==0)
  805. strcpy(rule_match,group_name);
  806. if(rule_match[0]==0)
  807. return(NULL);
  808. EncodePhonemes(rule_phonemes,buf,bad_phoneme);
  809. for(ix=0;; ix++)
  810. {
  811. if((c = buf[ix])==0) break;
  812. if(c==255)
  813. {
  814. fprintf(f_log,"%5d: Bad phoneme [%c] in %s",linenum,bad_phoneme[0],input);
  815. error_count++;
  816. break;
  817. }
  818. }
  819. strcpy(output,buf);
  820. len = strlen(buf)+1;
  821. len_name = strlen(group_name);
  822. if((len_name > 0) && (memcmp(rule_match,group_name,len_name) != 0))
  823. {
  824. utf8_in(&wc,rule_match,0);
  825. if((group_name[0] == '9') && IsDigit(wc))
  826. {
  827. // numeric group, rule_match starts with a digit, so OK
  828. }
  829. else
  830. {
  831. fprintf(f_log,"%5d: Wrong initial letters '%s' for group '%s'\n",linenum,rule_match,group_name);
  832. error_count++;
  833. }
  834. }
  835. strcpy(&output[len],rule_match);
  836. len += strlen(rule_match);
  837. if(debug_flag)
  838. {
  839. output[len] = RULE_LINENUM;
  840. output[len+1] = (linenum % 255) + 1;
  841. output[len+2] = (linenum / 255) + 1;
  842. len+=3;
  843. }
  844. if(rule_cond[0] != 0)
  845. {
  846. ix = -1;
  847. if(rule_cond[0] == '!')
  848. {
  849. // allow the rule only if the condition number is NOT set for the voice
  850. ix = atoi(&rule_cond[1]) + 32;
  851. }
  852. else
  853. {
  854. // allow the rule only if the condition number is set for the voice
  855. ix = atoi(rule_cond);
  856. }
  857. if((ix > 0) && (ix < 255))
  858. {
  859. output[len++] = RULE_CONDITION;
  860. output[len++] = ix;
  861. }
  862. else
  863. {
  864. fprintf(f_log,"%5d: bad condition number ?%d\n",linenum,ix);
  865. error_count++;
  866. }
  867. }
  868. if(rule_pre[0] != 0)
  869. {
  870. output[len++] = RULE_PRE;
  871. // output PRE string in reverse order
  872. for(ix = strlen(rule_pre)-1; ix>=0; ix--)
  873. output[len++] = rule_pre[ix];
  874. }
  875. if(rule_post[0] != 0)
  876. {
  877. sprintf(&output[len],"%c%s",RULE_POST,rule_post);
  878. len += (strlen(rule_post)+1);
  879. }
  880. output[len++]=0;
  881. prule = (char *)malloc(len);
  882. memcpy(prule,output,len);
  883. return(prule);
  884. } // end of compile_rule
  885. static int __cdecl string_sorter(char **a, char **b)
  886. {//=================================================
  887. char *pa, *pb;
  888. int ix;
  889. if((ix = strcmp(pa = *a,pb = *b)) != 0)
  890. return(ix);
  891. pa += (strlen(pa)+1);
  892. pb += (strlen(pb)+1);
  893. return(strcmp(pa,pb));
  894. } /* end of string_sorter */
  895. static int __cdecl rgroup_sorter(RGROUP *a, RGROUP *b)
  896. {//===================================================
  897. int ix;
  898. ix = strcmp(a->name,b->name);
  899. if(ix != 0) return(ix);
  900. return(a->start-b->start);
  901. }
  902. #ifdef OUTPUT_FORMAT
  903. void print_rule_group(FILE *f_out, int n_rules, char **rules, char *name)
  904. {//======================================================================
  905. int rule;
  906. int ix;
  907. unsigned char c;
  908. int len1;
  909. int len2;
  910. int spaces;
  911. char *p;
  912. char *pout;
  913. int condition;
  914. char buf[80];
  915. char suffix[12];
  916. static unsigned char symbols[] = {'@','&','%','+','#','$','D','Z','A','B','C','F'};
  917. fprintf(f_out,"\n$group %s\n",name);
  918. for(rule=0; rule<n_rules; rule++)
  919. {
  920. p = rules[rule];
  921. len1 = strlen(p) + 1;
  922. p = &p[len1];
  923. len2 = strlen(p);
  924. rule_match[0]=0;
  925. rule_pre[0]=0;
  926. rule_post[0]=0;
  927. condition = 0;
  928. pout = rule_match;
  929. for(ix=0; ix<len2; ix++)
  930. {
  931. switch(c = p[ix])
  932. {
  933. case RULE_PRE:
  934. *pout = 0;
  935. pout = rule_pre;
  936. break;
  937. case RULE_POST:
  938. *pout = 0;
  939. pout = rule_post;
  940. break;
  941. case RULE_CONDITION:
  942. condition = p[++ix];
  943. break;
  944. case RULE_ENDING:
  945. sprintf(suffix,"$%d[%x]",(p[ix+2]),p[ix+1] & 0x7f);
  946. ix += 2;
  947. strcpy(pout,suffix);
  948. pout += strlen(suffix);
  949. break;
  950. default:
  951. if(c <= RULE_LETTER7)
  952. c = symbols[c-RULE_SYLLABLE];
  953. if(c == ' ')
  954. c = '_';
  955. *pout++ = c;
  956. break;
  957. }
  958. }
  959. *pout = 0;
  960. spaces = 12;
  961. if(condition > 0)
  962. {
  963. sprintf(buf,"?%d ",condition);
  964. spaces -= strlen(buf);
  965. fprintf(f_out,"%s",buf);
  966. }
  967. if(rule_pre[0] != 0)
  968. {
  969. p = buf;
  970. for(ix=strlen(rule_pre)-1;ix>=0;ix--)
  971. *p++ = rule_pre[ix];
  972. sprintf(p,") ");
  973. spaces -= strlen(buf);
  974. for(ix=0; ix<spaces; ix++)
  975. fputc(' ',f_out);
  976. fprintf(f_out,"%s",buf);
  977. spaces = 0;
  978. }
  979. for(ix=0; ix<spaces; ix++)
  980. fputc(' ',f_out);
  981. spaces = 14;
  982. sprintf(buf," %s ",rule_match);
  983. if(rule_post[0] != 0)
  984. {
  985. p = &buf[strlen(buf)];
  986. sprintf(p,"(%s ",rule_post);
  987. }
  988. fprintf(f_out,"%s",buf);
  989. spaces -= strlen(buf);
  990. for(ix=0; ix<spaces; ix++)
  991. fputc(' ',f_out);
  992. DecodePhonemes(rules[rule],buf);
  993. fprintf(f_out,"%s\n",buf); // phonemes
  994. }
  995. }
  996. #endif
  997. //#define LIST_GROUP_INFO
  998. void output_rule_group(FILE *f_out, int n_rules, char **rules, char *name)
  999. {//=======================================================================
  1000. int ix;
  1001. int len1;
  1002. int len2;
  1003. int len_name;
  1004. char *p;
  1005. char *p2, *p3;
  1006. const char *common;
  1007. short nextchar_count[256];
  1008. memset(nextchar_count,0,sizeof(nextchar_count));
  1009. len_name = strlen(name);
  1010. #ifdef OUTPUT_FORMAT
  1011. print_rule_group(f_log,n_rules,rules,name);
  1012. #endif
  1013. // sort the rules in this group by their phoneme string
  1014. common = "";
  1015. qsort((void *)rules,n_rules,sizeof(char *),(int (__cdecl *)(const void *,const void *))string_sorter);
  1016. if(strcmp(name,"9")==0)
  1017. len_name = 0; // don't remove characters from numeric match strings
  1018. for(ix=0; ix<n_rules; ix++)
  1019. {
  1020. p = rules[ix];
  1021. len1 = strlen(p) + 1; // phoneme string
  1022. p3 = &p[len1];
  1023. p2 = p3 + len_name; // remove group name from start of match string
  1024. len2 = strlen(p2);
  1025. nextchar_count[(unsigned char)(p2[0])]++; // the next byte after the group name
  1026. if((common[0] != 0) && (strcmp(p,common)==0))
  1027. {
  1028. fwrite(p2,len2,1,f_out);
  1029. fputc(0,f_out); // no phoneme string, it's the same as previous rule
  1030. }
  1031. else
  1032. {
  1033. if((ix < n_rules-1) && (strcmp(p,rules[ix+1])==0))
  1034. {
  1035. common = rules[ix]; // phoneme string is same as next, set as common
  1036. fputc(RULE_PH_COMMON,f_out);
  1037. }
  1038. fwrite(p2,len2,1,f_out);
  1039. fputc(RULE_PHONEMES,f_out);
  1040. fwrite(p,len1,1,f_out);
  1041. }
  1042. }
  1043. #ifdef LIST_GROUP_INFO
  1044. for(ix=32; ix<256; ix++)
  1045. {
  1046. if(nextchar_count[ix] > 30)
  1047. printf("Group %s %c %d\n",name,ix,nextchar_count[ix]);
  1048. }
  1049. #endif
  1050. } // end of output_rule_group
  1051. static int compile_lettergroup(char *input, FILE *f_out)
  1052. {//=====================================================
  1053. char *p;
  1054. int group;
  1055. p = input;
  1056. if(!isdigit(p[0]) || !isdigit(p[1]))
  1057. {
  1058. return(1);
  1059. }
  1060. group = atoi(&p[1]);
  1061. if(group >= N_LETTER_GROUPS)
  1062. return(1);
  1063. while(!isspace2(*p)) p++;
  1064. fputc(RULE_GROUP_START,f_out);
  1065. fputc(RULE_LETTERGP2,f_out);
  1066. fputc(group + 'A', f_out);
  1067. for(;;)
  1068. {
  1069. while(isspace2(*p)) p++;
  1070. if(*p == 0)
  1071. break;
  1072. while((*p & 0xff) > ' ')
  1073. {
  1074. fputc(*p++, f_out);
  1075. }
  1076. fputc(0,f_out);
  1077. }
  1078. fputc(RULE_GROUP_END,f_out);
  1079. return(0);
  1080. }
  1081. static int compile_dictrules(FILE *f_in, FILE *f_out, char *fname_temp)
  1082. {//====================================================================
  1083. char *prule;
  1084. unsigned char *p;
  1085. int ix;
  1086. int c;
  1087. int gp;
  1088. FILE *f_temp;
  1089. int n_rules=0;
  1090. int count=0;
  1091. int different;
  1092. const char *prev_rgroup_name;
  1093. unsigned int char_code;
  1094. int compile_mode=0;
  1095. char *buf;
  1096. char buf1[200];
  1097. char *rules[N_RULES];
  1098. int n_rgroups = 0;
  1099. RGROUP rgroup[N_RULE_GROUP2];
  1100. linenum = 0;
  1101. group_name[0] = 0;
  1102. if((f_temp = fopen_log(fname_temp,"wb")) == NULL)
  1103. return(1);
  1104. for(;;)
  1105. {
  1106. linenum++;
  1107. buf = fgets(buf1,sizeof(buf1),f_in);
  1108. if(buf != NULL)
  1109. {
  1110. if((p = (unsigned char *)strstr(buf,"//")) != NULL)
  1111. *p = 0;
  1112. if(buf[0] == '\r') buf++; // ignore extra \r in \r\n
  1113. }
  1114. if((buf == NULL) || (buf[0] == '.'))
  1115. {
  1116. // next .group or end of file, write out the previous group
  1117. if(n_rules > 0)
  1118. {
  1119. strcpy(rgroup[n_rgroups].name,group_name);
  1120. rgroup[n_rgroups].start = ftell(f_temp);
  1121. output_rule_group(f_temp,n_rules,rules,group_name);
  1122. rgroup[n_rgroups].length = ftell(f_temp) - rgroup[n_rgroups].start;
  1123. n_rgroups++;
  1124. count += n_rules;
  1125. }
  1126. n_rules = 0;
  1127. if(compile_mode == 2)
  1128. {
  1129. // end of the character replacements section
  1130. fwrite(&n_rules,1,4,f_out); // write a zero word to terminate the replacemenmt list
  1131. compile_mode = 0;
  1132. }
  1133. if(buf == NULL) break; // end of file
  1134. if(memcmp(buf,".L",2)==0)
  1135. {
  1136. if(compile_lettergroup(&buf[2], f_out) != 0)
  1137. {
  1138. fprintf(f_log,"%5d: Bad lettergroup\n",linenum);
  1139. error_count++;
  1140. }
  1141. continue;
  1142. }
  1143. if(memcmp(buf,".replace",8)==0)
  1144. {
  1145. compile_mode = 2;
  1146. fputc(RULE_GROUP_START,f_out);
  1147. fputc(RULE_REPLACEMENTS,f_out);
  1148. // advance to next word boundary
  1149. while((ftell(f_out) & 3) != 0)
  1150. fputc(0,f_out);
  1151. }
  1152. if(memcmp(buf,".group",6)==0)
  1153. {
  1154. compile_mode = 1;
  1155. p = (unsigned char *)&buf[6];
  1156. while((p[0]==' ') || (p[0]=='\t')) p++; // Note: Windows isspace(0xe1) gives TRUE !
  1157. ix = 0;
  1158. while((*p > ' ') && (ix < LEN_GROUP_NAME))
  1159. group_name[ix++] = *p++;
  1160. group_name[ix]=0;
  1161. if(sscanf(group_name,"0x%x",&char_code)==1)
  1162. {
  1163. // group character is given as a character code (max 16 bits)
  1164. p = (unsigned char *)group_name;
  1165. if(char_code > 0x100)
  1166. {
  1167. *p++ = (char_code >> 8);
  1168. }
  1169. *p++ = char_code;
  1170. *p = 0;
  1171. }
  1172. if(strlen(group_name) > 2)
  1173. {
  1174. if(utf8_in(&c,group_name,0) < 2)
  1175. {
  1176. fprintf(f_log,"%5d: Group name longer than 2 bytes (UTF8)",linenum);
  1177. error_count++;
  1178. }
  1179. group_name[2] = 0;
  1180. }
  1181. }
  1182. continue;
  1183. }
  1184. switch(compile_mode)
  1185. {
  1186. case 1: // .group
  1187. prule = compile_rule(buf);
  1188. if((prule != NULL) && (n_rules < N_RULES))
  1189. {
  1190. rules[n_rules++] = prule;
  1191. }
  1192. break;
  1193. case 2: // .replace
  1194. {
  1195. int replace1;
  1196. int replace2;
  1197. char *p;
  1198. p = buf;
  1199. replace1 = 0;
  1200. replace2 = 0;
  1201. while(isspace2(*p)) p++;
  1202. ix = 0;
  1203. while((unsigned char)(*p) > 0x20) // not space or zero-byte
  1204. {
  1205. p += utf8_in(&c,p,0);
  1206. replace1 += (c << ix);
  1207. ix += 16;
  1208. }
  1209. while(isspace2(*p)) p++;
  1210. ix = 0;
  1211. while((unsigned char)(*p) > 0x20)
  1212. {
  1213. p += utf8_in(&c,p,0);
  1214. replace2 += (c << ix);
  1215. ix += 16;
  1216. }
  1217. if(replace1 != 0)
  1218. {
  1219. Write4Bytes(f_out,replace1); // write as little-endian
  1220. Write4Bytes(f_out,replace2); // if big-endian, reverse the bytes in LoadDictionary()
  1221. }
  1222. }
  1223. break;
  1224. }
  1225. }
  1226. fclose(f_temp);
  1227. qsort((void *)rgroup,n_rgroups,sizeof(rgroup[0]),(int (__cdecl *)(const void *,const void *))rgroup_sorter);
  1228. if((f_temp = fopen(fname_temp,"rb"))==NULL)
  1229. return(2);
  1230. prev_rgroup_name = "\n";
  1231. for(gp = 0; gp < n_rgroups; gp++)
  1232. {
  1233. fseek(f_temp,rgroup[gp].start,SEEK_SET);
  1234. if((different = strcmp(rgroup[gp].name, prev_rgroup_name)) != 0)
  1235. {
  1236. // not the same as the previous group
  1237. if(gp > 0)
  1238. fputc(RULE_GROUP_END,f_out);
  1239. fputc(RULE_GROUP_START,f_out);
  1240. fprintf(f_out, prev_rgroup_name = rgroup[gp].name);
  1241. fputc(0,f_out);
  1242. }
  1243. for(ix=rgroup[gp].length; ix>0; ix--)
  1244. {
  1245. c = fgetc(f_temp);
  1246. fputc(c,f_out);
  1247. }
  1248. if(different)
  1249. {
  1250. }
  1251. }
  1252. fputc(RULE_GROUP_END,f_out);
  1253. fputc(0,f_out);
  1254. fclose(f_temp);
  1255. remove(fname_temp);
  1256. fprintf(f_log,"\t%d rules, %d groups\n\n",count,n_rgroups);
  1257. return(0);
  1258. } // end of compile_dictrules
  1259. int CompileDictionary(const char *dsource, const char *dict_name, FILE *log, char *fname_err, int flags)
  1260. {//=====================================================================================================
  1261. // fname: space to write the filename in case of error
  1262. // flags: bit 0: include source line number information, for debug purposes.
  1263. FILE *f_in;
  1264. FILE *f_out;
  1265. int offset_rules=0;
  1266. int value;
  1267. char fname_in[sizeof(path_home)+45];
  1268. char fname_out[sizeof(path_home)+15];
  1269. char fname_temp[sizeof(path_home)+15];
  1270. char path[sizeof(path_home)+40]; // path_dsource+20
  1271. error_count = 0;
  1272. debug_flag = flags & 1;
  1273. if(dsource == NULL)
  1274. dsource = "";
  1275. f_log = log;
  1276. //f_log = fopen("log2.txt","w");
  1277. if(f_log == NULL)
  1278. f_log = stderr;
  1279. sprintf(path,"%s%s_",dsource,dict_name);
  1280. sprintf(fname_in,"%srules",path);
  1281. f_in = fopen_log(fname_in,"r");
  1282. if(f_in == NULL)
  1283. {
  1284. if(fname_err)
  1285. strcpy(fname_err,fname_in);
  1286. return(-1);
  1287. }
  1288. sprintf(fname_out,"%s%c%s_dict",path_home,PATHSEP,dict_name);
  1289. if((f_out = fopen_log(fname_out,"wb+")) == NULL)
  1290. {
  1291. if(fname_err)
  1292. strcpy(fname_err,fname_in);
  1293. return(-1);
  1294. }
  1295. sprintf(fname_temp,"%s%ctemp",path_home,PATHSEP);
  1296. transpose_offset = 0;
  1297. if(strcmp(dict_name,"ru") == 0)
  1298. {
  1299. // transpose cyrillic alphabet from unicode to iso8859-5
  1300. // transpose_offset = 0x430-0xd0;
  1301. transpose_offset = 0x42f; // range 0x01 to 0x22
  1302. transpose_min = 0x430;
  1303. transpose_max = 0x451;
  1304. }
  1305. value = N_HASH_DICT;
  1306. Write4Bytes(f_out,value);
  1307. Write4Bytes(f_out,offset_rules);
  1308. compile_dictlist_start();
  1309. fprintf(f_log,"Using phonemetable: '%s'\n",PhonemeTabName());
  1310. compile_dictlist_file(path,"roots");
  1311. if(translator->langopts.listx)
  1312. {
  1313. compile_dictlist_file(path,"list");
  1314. compile_dictlist_file(path,"listx");
  1315. }
  1316. else
  1317. {
  1318. compile_dictlist_file(path,"listx");
  1319. compile_dictlist_file(path,"list");
  1320. }
  1321. compile_dictlist_file(path,"extra");
  1322. compile_dictlist_end(f_out);
  1323. offset_rules = ftell(f_out);
  1324. fprintf(f_log,"Compiling: '%s'\n",fname_in);
  1325. compile_dictrules(f_in,f_out,fname_temp);
  1326. fclose(f_in);
  1327. fseek(f_out,4,SEEK_SET);
  1328. Write4Bytes(f_out,offset_rules);
  1329. fclose(f_out);
  1330. translator->LoadDictionary(dict_name,0);
  1331. return(error_count);
  1332. } // end of compile_dictionary