/native/external/espeak/src/compiledict.cpp

   1/***************************************************************************
   2 *   Copyright (C) 2005 to 2007 by Jonathan Duddington                     *
   3 *   email: jonsd@users.sourceforge.net                                    *
   4 *                                                                         *
   5 *   This program is free software; you can redistribute it and/or modify  *
   6 *   it under the terms of the GNU General Public License as published by  *
   7 *   the Free Software Foundation; either version 3 of the License, or     *
   8 *   (at your option) any later version.                                   *
   9 *                                                                         *
  10 *   This program is distributed in the hope that it will be useful,       *
  11 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13 *   GNU General Public License for more details.                          *
  14 *                                                                         *
  15 *   You should have received a copy of the GNU General Public License     *
  16 *   along with this program; if not, write see:                           *
  17 *               <http://www.gnu.org/licenses/>.                           *
  18 ***************************************************************************/
  19
  20#include "StdAfx.h"
  21
  22#include <stdio.h>
  23#include <ctype.h>
  24#include <stdlib.h>
  25#include <string.h>
  26#include <wctype.h>
  27
  28#include "speak_lib.h"
  29#include "speech.h"
  30#include "phoneme.h"
  31#include "synthesize.h"
  32#include "translate.h"
  33
  34//#define OPT_FORMAT         // format the text and write formatted copy to Log file 
  35//#define OUTPUT_FORMAT
  36
  37extern void Write4Bytes(FILE *f, int value);
  38int HashDictionary(const char *string);
  39
  40static FILE *f_log = NULL;
  41extern char *dir_dictionary;
  42
  43int linenum;
  44static int error_count;
  45static int transpose_offset;  // transpose character range for LookupDictList()
  46static int transpose_min;
  47static int transpose_max;
  48static int text_mode = 0;
  49static int debug_flag = 0;
  50
  51int hash_counts[N_HASH_DICT];
  52char *hash_chains[N_HASH_DICT];
  53
  54MNEM_TAB mnem_flags[] = {
  55	// these in the first group put a value in bits0-3 of dictionary_flags
  56	{"$1", 0x41},           // stress on 1st syllable
  57	{"$2", 0x42},           // stress on 2nd syllable
  58	{"$3", 0x43},
  59	{"$4", 0x44},
  60	{"$5", 0x45},
  61	{"$6", 0x46},
  62	{"$7", 0x47},
  63	{"$u", 0x48},           // reduce to unstressed
  64	{"$u1", 0x49},
  65	{"$u2", 0x4a},
  66	{"$u3", 0x4b},
  67	{"$u+",  0x4c},           // reduce to unstressed, but stress at end of clause
  68	{"$u1+", 0x4d},
  69	{"$u2+", 0x4e},
  70	{"$u3+", 0x4f},
  71
  72
  73	// these set the corresponding numbered bit if dictionary_flags
  74	{"$pause",     8},    /* ensure pause before this word */
  75	{"$only",      9},    /* only match on this word without suffix */
  76	{"$onlys",     10},    /* only match with none, or with 's' suffix */
  77	{"$strend",    11},    /* full stress if at end of clause */
  78	{"$strend2",   12},    /* full stress if at end of clause, or only followed by unstressed */
  79	{"$unstressend",13},   /* reduce stress at end of clause */
  80	{"$atend",     14},    /* use this pronunciation if at end of clause */
  81
  82	{"$dot",       16},   /* ignore '.' after this word (abbreviation) */
  83	{"$abbrev",    17},    /* use this pronuciation rather than split into letters */
  84	{"$stem",      18},   // must have a suffix
  85
  86// language specific
  87	{"$double",    19},   // IT double the initial consonant of next word
  88	{"$alt",       20},   // use alternative pronunciation
  89	{"$alt2",      21},
  90	
  91
  92	{"$brk",       28},   // a shorter $pause
  93	{"$text",      29},   // word translates to replcement text, not phonemes
  94
  95// flags in dictionary word 2
  96	{"$verbf",   0x20},    /* verb follows */
  97	{"$verbsf",  0x21},    /* verb follows, allow -s suffix */
  98	{"$nounf",   0x22},    /* noun follows */
  99	{"$pastf",   0x23},   /* past tense follows */
 100	{"$verb",    0x24},   /* use this pronunciation when its a verb */
 101	{"$noun",    0x25},   /* use this pronunciation when its a noun */
 102	{"$past",    0x26},   /* use this pronunciation when its past tense */
 103	{"$verbextend",0x28},   /* extend influence of 'verb follows' */
 104	{"$capital", 0x29},   /* use this pronunciation if initial letter is upper case */
 105	{"$allcaps", 0x2a},   /* use this pronunciation if initial letter is upper case */
 106	{"$accent",  0x2b},   // character name is base-character name + accent name
 107
 108	// doesn't set dictionary_flags
 109	{"$?",        100},   // conditional rule, followed by byte giving the condition number
 110
 111	{"$textmode",  200},
 112	{"$phonememode", 201},
 113	{NULL,   -1}
 114};
 115
 116
 117#define LEN_GROUP_NAME  12
 118
 119typedef struct {
 120	char name[LEN_GROUP_NAME+1];
 121	unsigned int start;
 122	unsigned int length;
 123} RGROUP;
 124
 125
 126int isspace2(unsigned int c)
 127{//=========================
 128// can't use isspace() because on Windows, isspace(0xe1) gives TRUE !
 129	int c2;
 130
 131	if(((c2 = (c & 0xff)) == 0) || (c > ' '))
 132		return(0);
 133	return(1);
 134}
 135
 136
 137static FILE *fopen_log(const char *fname,const char *access)
 138{//==================================================
 139// performs fopen, but produces error message to f_log if it fails
 140	FILE *f;
 141
 142	if((f = fopen(fname,access)) == NULL)
 143	{
 144		if(f_log != NULL)
 145			fprintf(f_log,"Can't access (%s) file '%s'\n",access,fname);
 146	}
 147	return(f);
 148}
 149
 150
 151#ifdef OPT_FORMAT
 152static const char *lookup_mnem(MNEM_TAB *table, int value)
 153//========================================================
 154/* Lookup a mnemonic string in a table, return its name */
 155{
 156   while(table->mnem != NULL)
 157   {
 158      if(table->value==value)
 159         return(table->mnem);
 160      table++;
 161   }
 162   return("??");   /* not found */
 163}   /* end of mnem */
 164#endif
 165
 166
 167
 168
 169int compile_line(char *linebuf, char *dict_line, int *hash)
 170{//========================================================
 171// Compile a line in the language_list file
 172	unsigned char  c;
 173	char *p;
 174	char *word;
 175	char *phonetic;
 176	unsigned int  ix;
 177	int  step;
 178	unsigned int  n_flag_codes = 0;
 179	int  flag_offset;
 180	int  length;
 181	int  multiple_words = 0;
 182	char *multiple_string = NULL;
 183	char *multiple_string_end = NULL;
 184	
 185	int len_word;
 186	int len_phonetic;
 187	int text_not_phonemes;   // this word specifies replacement text, not phonemes
 188	unsigned int  wc;
 189	
 190	char *mnemptr;
 191	char *comment;
 192	unsigned char flag_codes[100];
 193	char encoded_ph[200];
 194	unsigned char bad_phoneme[4];
 195static char nullstring[] = {0};
 196
 197	comment = NULL;
 198	text_not_phonemes = 0;
 199	phonetic = word = nullstring;
 200
 201	p = linebuf;
 202//	while(isspace2(*p)) p++;
 203
 204#ifdef deleted
 205	if(*p == '$')
 206	{
 207		if(memcmp(p,"$textmode",9) == 0)
 208		{
 209			text_mode = 1;
 210			return(0);
 211		}
 212		if(memcmp(p,"$phonememode",12) == 0)
 213		{
 214			text_mode = 0;
 215			return(0);
 216		}
 217	}
 218#endif
 219
 220	step = 0;
 221	
 222	c = 0;
 223	while(c != '\n')
 224	{
 225		c = *p;
 226	
 227		if((c == '?') && (step==0))
 228		{
 229			// conditional rule, allow only if the numbered condition is set for the voice
 230			flag_offset = 100;
 231
 232			p++;
 233			if(*p == '!')
 234			{
 235				// allow only if the numbered condition is NOT set
 236				flag_offset = 132;
 237				p++;
 238			}
 239
 240			ix = 0;
 241			if(isdigit(*p))
 242			{
 243				ix += (*p-'0');
 244				p++;
 245			}
 246			if(isdigit(*p))
 247			{
 248				ix = ix*10 + (*p-'0');
 249				p++;
 250			}
 251			flag_codes[n_flag_codes++] = ix + flag_offset;
 252			c = *p;
 253		}
 254		
 255		if((c == '$') && isalnum(p[1]))
 256		{
 257			/* read keyword parameter */
 258			mnemptr = p;
 259			while(!isspace2(c = *p)) p++;
 260			*p = 0;
 261	
 262			ix = LookupMnem(mnem_flags,mnemptr);
 263			if(ix > 0)
 264			{
 265				if(ix == 200)
 266				{
 267					text_mode = 1;
 268				}
 269				else
 270				if(ix == 201)
 271				{
 272					text_mode = 0;
 273				}
 274				else
 275				if(ix == BITNUM_FLAG_TEXTMODE)
 276				{
 277					text_not_phonemes = 1;
 278				}
 279				else
 280				{
 281					flag_codes[n_flag_codes++] = ix;
 282				}
 283			}
 284			else
 285			{
 286				fprintf(f_log,"%5d: Unknown keyword: %s\n",linenum,mnemptr);
 287				error_count++;
 288			}
 289		}
 290	
 291		if((c == '/') && (p[1] == '/') && (multiple_words==0))
 292		{
 293			c = '\n';   /* "//" treat comment as end of line */
 294			comment = p;
 295		}
 296	
 297		switch(step)
 298		{
 299		case 0:
 300			if(c == '(')
 301			{
 302				multiple_words = 1;
 303				word = p+1;
 304				step = 1;
 305			}
 306			else
 307			if(!isspace2(c))
 308			{
 309				word = p;
 310				step = 1;
 311			}
 312			break;
 313	
 314		case 1:
 315			if(isspace2(c))
 316			{
 317				p[0] = 0;   /* terminate english word */
 318
 319				if(multiple_words)
 320				{
 321					multiple_string = multiple_string_end = p+1;
 322					step = 2;
 323				}
 324				else
 325				{
 326					step = 3;
 327				}
 328			}
 329			else
 330			if((c == ')') && multiple_words)
 331			{
 332				p[0] = 0;
 333				step = 3;
 334				multiple_words = 0;
 335			}
 336			break;
 337
 338		case 2:
 339			if(isspace2(c))
 340			{
 341				multiple_words++;
 342			}
 343			else
 344			if(c == ')')
 345			{
 346				p[0] = ' ';   // terminate extra string
 347				multiple_string_end = p+1;
 348				step = 3;
 349			}
 350			break;
 351	
 352		case 3:
 353			if(!isspace2(c))
 354			{
 355				phonetic = p;
 356				step = 4;
 357			}
 358			break;
 359	
 360		case 4:
 361			if(isspace2(c))
 362			{
 363				p[0] = 0;   /* terminate phonetic */
 364				step = 5;
 365			}
 366			break;
 367	
 368		case 5:
 369			break;
 370		}
 371		p++;
 372	}
 373	
 374	if(word[0] == 0)
 375	{
 376#ifdef OPT_FORMAT
 377		if(comment != NULL)
 378			fprintf(f_log,"%s",comment);
 379		else
 380			fputc('\n',f_log);
 381#endif
 382		return(0);   /* blank line */
 383	}
 384
 385	if(text_mode)
 386		text_not_phonemes = 1;
 387
 388	if(text_not_phonemes != translator->langopts.textmode)
 389	{
 390		flag_codes[n_flag_codes++] = BITNUM_FLAG_TEXTMODE;
 391	}
 392
 393	if(text_not_phonemes)
 394	{
 395		// this is replacement text, so don't encode as phonemes. Restrict the length of the replacement word
 396		strncpy0(encoded_ph,phonetic,N_WORD_BYTES-4);
 397	}
 398	else
 399	{
 400		EncodePhonemes(phonetic,encoded_ph,bad_phoneme);
 401		if(strchr(encoded_ph,phonSWITCH) != 0)
 402		{
 403			flag_codes[n_flag_codes++] = BITNUM_FLAG_ONLY_S;  // don't match on suffixes (except 's') when switching languages
 404		}
 405
 406		// check for errors in the phonemes codes
 407		for(ix=0; ix<sizeof(encoded_ph); ix++)
 408		{
 409			c = encoded_ph[ix];
 410			if(c == 0)   break;
 411		
 412			if(c == 255)
 413			{
 414				/* unrecognised phoneme, report error */
 415				fprintf(f_log,"%5d: Bad phoneme [%c] (0x%x) in: %s  %s\n",linenum,bad_phoneme[0],bad_phoneme[0],word,phonetic);
 416				error_count++;
 417			}
 418		}
 419	}
 420
 421	if(sscanf(word,"U+%x",&wc) == 1)
 422	{
 423		// Character code
 424		ix = utf8_out(wc, word);
 425		word[ix] = 0;
 426	}
 427	else
 428	if((word[0] & 0x80)==0)  // 7 bit ascii only
 429	{
 430		// If first letter is uppercase, convert to lower case.  (Only if it's 7bit ascii)
 431		// ??? need to consider utf8 here
 432		word[0] = tolower(word[0]);
 433	}
 434
 435	len_word = strlen(word);
 436
 437	if(transpose_offset > 0)
 438	{
 439		len_word = TransposeAlphabet(word, transpose_offset, transpose_min, transpose_max);
 440	}
 441
 442	*hash = HashDictionary(word);
 443	len_phonetic = strlen(encoded_ph);
 444	
 445	dict_line[1] = len_word;   // bit 6 indicates whether the word has been compressed
 446	len_word &= 0x3f;
 447
 448	memcpy(&dict_line[2],word,len_word);
 449
 450	if(len_phonetic == 0)
 451	{
 452		// no phonemes specified. set bit 7
 453		dict_line[1] |= 0x80;
 454		length = len_word + 2;
 455	}
 456	else
 457	{
 458		length = len_word + len_phonetic + 3;
 459		strcpy(&dict_line[(len_word)+2],encoded_ph);
 460	}
 461	
 462	for(ix=0; ix<n_flag_codes; ix++)
 463	{
 464		dict_line[ix+length] = flag_codes[ix];
 465	}
 466	length += n_flag_codes;
 467
 468	if((multiple_string != NULL) && (multiple_words > 0))
 469	{
 470		if(multiple_words > 10)
 471		{
 472			fprintf(f_log,"%5d: Two many parts in a multi-word entry: %d\n",linenum,multiple_words);
 473		}
 474		else
 475		{
 476			dict_line[length++] = 80 + multiple_words;
 477			ix = multiple_string_end - multiple_string;
 478			memcpy(&dict_line[length],multiple_string,ix);
 479			length += ix;
 480		}
 481	}
 482	dict_line[0] = length;
 483
 484#ifdef OPT_FORMAT
 485	spaces = 16;
 486	for(ix=0; ix<n_flag_codes; ix++)
 487	{
 488		if(flag_codes[ix] >= 100)
 489		{
 490			fprintf(f_log,"?%d ",flag_codes[ix]-100);
 491			spaces -= 3;
 492		}
 493	}
 494
 495	fprintf(f_log,"%s",word);
 496	spaces -= strlen(word);
 497	DecodePhonemes(encoded_ph,decoded_ph);
 498	while(spaces-- > 0) fputc(' ',f_log);
 499	spaces += (14 - strlen(decoded_ph));
 500	
 501	fprintf(f_log," %s",decoded_ph);
 502	while(spaces-- > 0) fputc(' ',f_log);
 503	for(ix=0; ix<n_flag_codes; ix++)
 504	{
 505		if(flag_codes[ix] < 100)
 506			fprintf(f_log," %s",lookup_mnem(mnem_flags,flag_codes[ix]));
 507	}
 508	if(comment != NULL)
 509		fprintf(f_log," %s",comment);
 510	else
 511		fputc('\n',f_log);
 512#endif
 513
 514	return(length);
 515}  /* end of compile_line */
 516
 517
 518
 519void compile_dictlist_start(void)
 520{//==============================
 521// initialise dictionary list
 522	int ix;
 523	char *p;
 524	char *p2;
 525
 526	for(ix=0; ix<N_HASH_DICT; ix++)
 527	{
 528		p = hash_chains[ix];
 529		while(p != NULL)
 530		{
 531			memcpy(&p2,p,sizeof(char *));
 532			free(p);
 533			p = p2;
 534		}
 535		hash_chains[ix] = NULL;
 536		hash_counts[ix]=0;
 537	}
 538}
 539
 540
 541void compile_dictlist_end(FILE *f_out)
 542{//===================================
 543// Write out the compiled dictionary list
 544	int hash;
 545	int length;
 546	char *p;
 547
 548	if(f_log != NULL)
 549	{
 550#ifdef OUTPUT_FORMAT
 551		for(hash=0; hash<N_HASH_DICT; hash++)
 552		{
 553			fprintf(f_log,"%8d",hash_counts[hash]);
 554			if((hash & 7) == 7)
 555				fputc('\n',f_log);
 556		}
 557		fflush(f_log);
 558#endif
 559	}
 560	
 561	for(hash=0; hash<N_HASH_DICT; hash++)
 562	{
 563		p = hash_chains[hash];
 564		hash_counts[hash] = (int)ftell(f_out);
 565	
 566		while(p != NULL)
 567		{
 568			length = *(p+sizeof(char *));
 569			fwrite(p+sizeof(char *),length,1,f_out);
 570			memcpy(&p,p,sizeof(char *));
 571		}
 572		fputc(0,f_out);
 573	}
 574}
 575
 576
 577
 578int compile_dictlist_file(const char *path, const char* filename)
 579{//==============================================================
 580	int  length;
 581	int  hash;
 582	char *p;
 583	int  count=0;
 584	FILE *f_in;
 585	char buf[200];
 586	char fname[sizeof(path_home)+45];
 587	char dict_line[128];
 588	
 589	text_mode = 0;
 590
 591	sprintf(fname,"%s%s",path,filename);
 592	if((f_in = fopen(fname,"r")) == NULL)
 593		return(-1);
 594
 595	fprintf(f_log,"Compiling: '%s'\n",fname);
 596
 597	linenum=0;
 598	
 599	while(fgets(buf,sizeof(buf),f_in) != NULL)
 600	{
 601		linenum++;
 602
 603		length = compile_line(buf,dict_line,&hash);
 604		if(length == 0)  continue;   /* blank line */
 605
 606		hash_counts[hash]++;
 607	
 608		p = (char *)malloc(length+sizeof(char *));
 609		if(p == NULL)
 610		{
 611			if(f_log != NULL)
 612			{
 613				fprintf(f_log,"Can't allocate memory\n");
 614				error_count++;
 615			}
 616			break;
 617		}
 618	
 619		memcpy(p,&hash_chains[hash],sizeof(char *));
 620		hash_chains[hash] = p;
 621		memcpy(p+sizeof(char *),dict_line,length);
 622		count++;
 623	}
 624	
 625	fprintf(f_log,"\t%d entries\n",count);
 626	fclose(f_in);
 627	return(0);
 628}   /* end of compile_dictlist_file */
 629
 630
 631
 632char rule_cond[80];
 633char rule_pre[80];
 634char rule_post[80];
 635char rule_match[80];
 636char rule_phonemes[80];
 637char group_name[LEN_GROUP_NAME+1];
 638
 639#define N_RULES 2000		// max rules for each group
 640
 641
 642int hexdigit(char c)
 643{//=================
 644	if(isdigit(c))
 645		return(c - '0');
 646	return(tolower(c) - 'a' + 10);
 647}
 648
 649
 650void copy_rule_string(char *string, int &state)
 651{//============================================
 652// state 0: conditional, 1=pre, 2=match, 3=post, 4=phonemes
 653	static char *outbuf[5] = {rule_cond, rule_pre, rule_match, rule_post, rule_phonemes};
 654	static int next_state[5] = {2,2,4,4,4};
 655	char *output;
 656	char *p;
 657	int ix;
 658	int len;
 659	char c;
 660	int  sxflags;
 661	int  value;
 662	int  literal;
 663
 664	if(string[0] == 0) return;
 665
 666	output = outbuf[state];
 667	if(state==4)
 668	{
 669		// append to any previous phoneme string, i.e. allow spaces in the phoneme string
 670		len = strlen(rule_phonemes);
 671		if(len > 0)
 672			rule_phonemes[len++] = ' ';
 673		output = &rule_phonemes[len];
 674	}
 675	sxflags = 0x808000;           // to ensure non-zero bytes
 676	
 677	for(p=string,ix=0;;)
 678	{
 679		literal = 0;
 680		c = *p++;
 681		if(c == '\\')
 682		{
 683			c = *p++;   // treat next character literally
 684			if((c >= '0') && (c <= '3') && (p[0] >= '0') && (p[0] <= '7') && (p[1] >= '0') && (p[1] <= '7'))
 685			{
 686				// character code given by 3 digit octal value;
 687				c = (c-'0')*64 + (p[0]-'0')*8 + (p[1]-'0');
 688				p += 2;
 689			}
 690			literal = 1;
 691		}
 692
 693		if((state==1) || (state==3))
 694		{
 695			// replace special characters (note: 'E' is reserved for a replaced silent 'e')
 696			if(literal == 0)
 697			{
 698				static const char lettergp_letters[9] = {LETTERGP_A,LETTERGP_B,LETTERGP_C,0,0,LETTERGP_F,LETTERGP_G,LETTERGP_H,LETTERGP_Y};
 699				switch(c)
 700				{
 701				case '_':
 702					c = RULE_SPACE;
 703					break;
 704
 705				case 'Y':
 706					c = 'I';   // drop through to next case
 707				case 'A':   // vowel
 708				case 'B':
 709				case 'C':
 710				case 'H':
 711				case 'F':
 712				case 'G':
 713					if(state == 1)
 714					{
 715						// pre-rule, put the number before the RULE_LETTERGP;
 716						output[ix++] = lettergp_letters[c-'A'] + 'A';
 717						c = RULE_LETTERGP;
 718					}
 719					else
 720					{
 721						output[ix++] = RULE_LETTERGP;
 722						c = lettergp_letters[c-'A'] + 'A';
 723					}
 724					break;
 725				case 'D':
 726					c = RULE_DIGIT;
 727					break;
 728				case 'K':
 729					c = RULE_NOTVOWEL;
 730					break;
 731				case 'N':
 732					c = RULE_NO_SUFFIX;
 733					break;
 734				case 'V':
 735					c = RULE_IFVERB;
 736					break;
 737				case 'Z':
 738					c = RULE_NONALPHA;
 739					break;
 740				case '+':
 741					c = RULE_INC_SCORE;
 742					break;
 743				case '@':
 744					c = RULE_SYLLABLE;
 745					break;
 746				case '&':
 747					c = RULE_STRESSED;
 748					break;
 749				case '%':
 750					c = RULE_DOUBLE;
 751					break;
 752				case '#':
 753					c = RULE_DEL_FWD;
 754					break;
 755				case '!':
 756					c = RULE_CAPITAL;
 757					break;
 758				case 'T':
 759					c = RULE_ALT1;
 760					break;
 761				case 'W':
 762					c = RULE_SPELLING;
 763					break;
 764				case 'X':
 765					c = RULE_NOVOWELS;
 766					break;
 767				case 'L':
 768					// expect two digits
 769					c = *p++ - '0';
 770					value = *p++ - '0';
 771					c = c * 10 + value;
 772					if((value < 0) || (value > 9) || (c <= 0) || (c >= N_LETTER_GROUPS))
 773					{
 774						c = 0;
 775						fprintf(f_log,"%5d: Expected 2 digits after 'L'",linenum);
 776						error_count++;
 777					}
 778					c += 'A';
 779					if(state == 1)
 780					{
 781						// pre-rule, put the group number before the RULE_LETTERGP command
 782						output[ix++] = c;
 783						c = RULE_LETTERGP2;
 784					}
 785					else
 786					{
 787						output[ix++] = RULE_LETTERGP2;
 788					}
 789					break;
 790
 791				case 'P':
 792					sxflags |= SUFX_P;   // Prefix, now drop through to Suffix
 793				case '$':   // obsolete, replaced by S
 794				case 'S':
 795					output[ix++] = RULE_ENDING;
 796					value = 0;
 797					while(!isspace2(c = *p++) && (c != 0))
 798					{
 799						switch(c)
 800						{
 801						case 'e':
 802							sxflags |= SUFX_E;
 803							break;
 804						case 'i':
 805							sxflags |= SUFX_I;
 806							break;
 807						case 'p':	// obsolete, replaced by 'P' above
 808							sxflags |= SUFX_P;
 809							break;
 810						case 'v':
 811							sxflags |= SUFX_V;
 812							break;
 813						case 'd':
 814							sxflags |= SUFX_D;
 815							break;
 816						case 'f':
 817							sxflags |= SUFX_F;
 818							break;
 819						case 'q':
 820							sxflags |= SUFX_Q;
 821							break;
 822						case 't':
 823							sxflags |= SUFX_T;
 824							break;
 825						case 'b':
 826							sxflags |= SUFX_B;
 827							break;
 828						default:
 829							if(isdigit(c))
 830								value = (value*10) + (c - '0');
 831							break;
 832						}
 833					}
 834					p--;
 835					output[ix++] = sxflags >> 16;
 836					output[ix++] = sxflags >> 8;
 837					c = value | 0x80;
 838					break;
 839				}
 840			}
 841		}
 842		output[ix++] = c;
 843		if(c == 0) break;
 844	}
 845
 846	state = next_state[state];
 847}  //  end of copy_rule_string
 848
 849
 850
 851char *compile_rule(char *input)
 852{//============================
 853	int ix;
 854	unsigned char c;
 855	int wc;
 856	char *p;
 857	char *prule;
 858	int len;
 859	int len_name;
 860	int state=2;
 861	int finish=0;
 862	int pre_bracket=0;
 863	char buf[80];
 864	char output[150];
 865	unsigned char bad_phoneme[4];
 866
 867	buf[0]=0;
 868	rule_cond[0]=0;
 869	rule_pre[0]=0;
 870	rule_post[0]=0;
 871	rule_match[0]=0;
 872	rule_phonemes[0]=0;
 873
 874	p = buf;
 875	
 876	for(ix=0; finish==0; ix++)
 877	{
 878		c = input[ix];
 879
 880		switch(c = input[ix])
 881		{
 882		case ')':		// end of prefix section
 883			*p = 0;
 884			state = 1;
 885			pre_bracket = 1;
 886			copy_rule_string(buf,state);
 887			p = buf;
 888			break;
 889			
 890		case '(':		// start of suffix section
 891			*p = 0;
 892			state = 2;
 893			copy_rule_string(buf,state);
 894			state = 3;
 895			p = buf;
 896			break;
 897			
 898		case '\n':		// end of line
 899		case '\r':
 900		case 0:			// end of line
 901			*p = 0;
 902			copy_rule_string(buf,state);
 903			finish=1;
 904			break;
 905			
 906		case '\t':		// end of section section
 907		case ' ':
 908			*p = 0;
 909			copy_rule_string(buf,state);
 910			p = buf;
 911			break;
 912			
 913		case '?':
 914			if(state==2)
 915				state=0;
 916			else
 917				*p++ = c;
 918			break;
 919
 920		default:
 921			*p++ = c;
 922			break;
 923		}
 924	}
 925	
 926	if(strcmp(rule_match,"$group")==0)
 927		strcpy(rule_match,group_name);
 928
 929	if(rule_match[0]==0)
 930		return(NULL);
 931
 932	EncodePhonemes(rule_phonemes,buf,bad_phoneme);
 933	for(ix=0;; ix++)
 934	{
 935		if((c = buf[ix])==0) break;
 936		if(c==255)
 937		{
 938			fprintf(f_log,"%5d: Bad phoneme [%c] in %s",linenum,bad_phoneme[0],input);
 939			error_count++;
 940			break;
 941		}
 942	}
 943	strcpy(output,buf);
 944	len = strlen(buf)+1;
 945	
 946	len_name = strlen(group_name);
 947	if((len_name > 0) && (memcmp(rule_match,group_name,len_name) != 0))
 948	{
 949		utf8_in(&wc,rule_match,0);
 950		if((group_name[0] == '9') && IsDigit(wc))
 951		{
 952			// numeric group, rule_match starts with a digit, so OK
 953		}
 954		else
 955		{
 956			fprintf(f_log,"%5d: Wrong initial letters '%s' for group '%s'\n",linenum,rule_match,group_name);
 957			error_count++;
 958		}
 959	}
 960	strcpy(&output[len],rule_match);
 961	len += strlen(rule_match);
 962
 963	if(debug_flag)
 964	{
 965		output[len] = RULE_LINENUM;
 966		output[len+1] = (linenum % 255) + 1;
 967		output[len+2] = (linenum / 255) + 1;
 968		len+=3;
 969	}
 970
 971	if(rule_cond[0] != 0)
 972	{
 973		ix = -1;
 974		if(rule_cond[0] == '!')
 975		{
 976			// allow the rule only if the condition number is NOT set for the voice
 977			ix = atoi(&rule_cond[1]) + 32;
 978		}
 979		else
 980		{
 981			// allow the rule only if the condition number is set for the voice
 982			ix = atoi(rule_cond);
 983		}
 984
 985		if((ix > 0) && (ix < 255))
 986		{
 987			output[len++] = RULE_CONDITION;
 988			output[len++] = ix;
 989		}
 990		else
 991		{
 992			fprintf(f_log,"%5d: bad condition number ?%d\n",linenum,ix);
 993			error_count++;
 994		}
 995	}
 996	if(rule_pre[0] != 0)
 997	{
 998		output[len++] = RULE_PRE;
 999		// output PRE string in reverse order
1000		for(ix = strlen(rule_pre)-1; ix>=0; ix--)
1001			output[len++] = rule_pre[ix];
1002	}
1003
1004	if(rule_post[0] != 0)
1005	{
1006		sprintf(&output[len],"%c%s",RULE_POST,rule_post);
1007		len += (strlen(rule_post)+1);
1008	}
1009	output[len++]=0;
1010	prule = (char *)malloc(len);
1011	memcpy(prule,output,len);
1012	return(prule);
1013}  //  end of compile_rule
1014
1015
1016static int __cdecl string_sorter(char **a, char **b)
1017{//=================================================
1018	char *pa, *pb;
1019	int ix;
1020
1021   if((ix = strcmp(pa = *a,pb = *b)) != 0)
1022	   return(ix);
1023	pa += (strlen(pa)+1);
1024	pb += (strlen(pb)+1);
1025   return(strcmp(pa,pb));
1026}   /* end of string_sorter */
1027
1028
1029static int __cdecl rgroup_sorter(RGROUP *a, RGROUP *b)
1030{//===================================================
1031	int ix;
1032	ix = strcmp(a->name,b->name);
1033	if(ix != 0) return(ix);
1034	return(a->start-b->start);
1035}
1036
1037
1038#ifdef OUTPUT_FORMAT
1039void print_rule_group(FILE *f_out, int n_rules, char **rules, char *name)
1040{//======================================================================
1041	int rule;
1042	int ix;
1043	unsigned char c;
1044	int len1;
1045	int len2;
1046	int spaces;
1047	char *p;
1048	char *pout;
1049	int condition;
1050	char buf[80];
1051	char suffix[12];
1052
1053	static unsigned char symbols[] = {'@','&','%','+','#','$','D','Z','A','B','C','F'};
1054
1055	fprintf(f_out,"\n$group %s\n",name);
1056
1057	for(rule=0; rule<n_rules; rule++)
1058	{
1059		p = rules[rule];
1060		len1 = strlen(p) + 1;
1061		p = &p[len1];
1062		len2 = strlen(p);
1063		
1064		rule_match[0]=0;
1065		rule_pre[0]=0;
1066		rule_post[0]=0;
1067		condition = 0;
1068
1069		pout = rule_match;
1070		for(ix=0; ix<len2; ix++)
1071		{
1072			switch(c = p[ix])
1073			{
1074			case RULE_PRE:
1075				*pout = 0;
1076				pout = rule_pre;
1077				break;
1078			case RULE_POST:
1079				*pout = 0;
1080				pout = rule_post;
1081				break;
1082			case RULE_CONDITION:
1083				condition = p[++ix];
1084				break;
1085			case RULE_ENDING:
1086				sprintf(suffix,"$%d[%x]",(p[ix+2]),p[ix+1] & 0x7f);
1087				ix += 2;
1088				strcpy(pout,suffix);
1089				pout += strlen(suffix);
1090				break;
1091			default:
1092				if(c <= RULE_LETTER7)
1093					c = symbols[c-RULE_SYLLABLE];
1094				if(c == ' ')
1095					c = '_';
1096				*pout++ = c;
1097				break;
1098			}
1099		}
1100		*pout = 0;
1101		
1102		spaces = 12;
1103		if(condition > 0)
1104		{
1105			sprintf(buf,"?%d ",condition);
1106			spaces -= strlen(buf);
1107			fprintf(f_out,"%s",buf);
1108		}
1109
1110		if(rule_pre[0] != 0)
1111		{
1112			p = buf;
1113			for(ix=strlen(rule_pre)-1;ix>=0;ix--)
1114				*p++ = rule_pre[ix];
1115			sprintf(p,") ");
1116			spaces -= strlen(buf);
1117			for(ix=0; ix<spaces; ix++)
1118			   fputc(' ',f_out);
1119			fprintf(f_out,"%s",buf);
1120			spaces = 0;
1121		}
1122		
1123		for(ix=0; ix<spaces; ix++)
1124			fputc(' ',f_out);
1125		
1126		spaces = 14;
1127		sprintf(buf," %s ",rule_match);
1128		if(rule_post[0] != 0)
1129		{
1130			p = &buf[strlen(buf)];
1131			sprintf(p,"(%s ",rule_post);
1132		}
1133		fprintf(f_out,"%s",buf);
1134		spaces -= strlen(buf);
1135
1136		for(ix=0; ix<spaces; ix++)
1137			fputc(' ',f_out);
1138		DecodePhonemes(rules[rule],buf);
1139		fprintf(f_out,"%s\n",buf);   // phonemes
1140	}
1141}
1142#endif
1143
1144
1145//#define LIST_GROUP_INFO
1146void output_rule_group(FILE *f_out, int n_rules, char **rules, char *name)
1147{//=======================================================================
1148	int ix;
1149	int len1;
1150	int len2;
1151	int len_name;
1152	char *p;
1153	char *p2, *p3;
1154	const char *common;
1155
1156	short nextchar_count[256];
1157	memset(nextchar_count,0,sizeof(nextchar_count));
1158
1159	len_name = strlen(name);
1160
1161#ifdef OUTPUT_FORMAT
1162	print_rule_group(f_log,n_rules,rules,name);
1163#endif
1164
1165	// sort the rules in this group by their phoneme string
1166	common = "";
1167	qsort((void *)rules,n_rules,sizeof(char *),(int (__cdecl *)(const void *,const void *))string_sorter);
1168
1169	if(strcmp(name,"9")==0)
1170		len_name = 0;    //  don't remove characters from numeric match strings
1171
1172	for(ix=0; ix<n_rules; ix++)
1173	{
1174		p = rules[ix];
1175		len1 = strlen(p) + 1;  // phoneme string
1176		p3 = &p[len1];
1177		p2 = p3 + len_name;        // remove group name from start of match string
1178		len2 = strlen(p2);
1179
1180		nextchar_count[(unsigned char)(p2[0])]++;   // the next byte after the group name
1181
1182		if((common[0] != 0) && (strcmp(p,common)==0))
1183		{
1184			fwrite(p2,len2,1,f_out);
1185			fputc(0,f_out);		// no phoneme string, it's the same as previous rule
1186		}
1187		else
1188		{
1189			if((ix < n_rules-1) && (strcmp(p,rules[ix+1])==0))
1190			{
1191				common = rules[ix];   // phoneme string is same as next, set as common
1192				fputc(RULE_PH_COMMON,f_out);
1193			}
1194
1195			fwrite(p2,len2,1,f_out);
1196			fputc(RULE_PHONEMES,f_out);
1197			fwrite(p,len1,1,f_out);
1198		}
1199	}
1200
1201#ifdef LIST_GROUP_INFO
1202	for(ix=32; ix<256; ix++)
1203	{
1204		if(nextchar_count[ix] > 30)
1205			printf("Group %s   %c  %d\n",name,ix,nextchar_count[ix]);
1206	}
1207#endif
1208}  //  end of output_rule_group
1209
1210
1211
1212static int compile_lettergroup(char *input, FILE *f_out)
1213{//=====================================================
1214	char *p;
1215	int group;
1216
1217	p = input;
1218	if(!isdigit(p[0]) || !isdigit(p[1]))
1219	{
1220		return(1);
1221	}
1222
1223	group = atoi(&p[1]);
1224	if(group >= N_LETTER_GROUPS)
1225		return(1);
1226
1227	while(!isspace2(*p)) p++;
1228
1229	fputc(RULE_GROUP_START,f_out);
1230	fputc(RULE_LETTERGP2,f_out);
1231	fputc(group + 'A', f_out);
1232
1233	for(;;)
1234	{
1235		while(isspace2(*p)) p++;
1236		if(*p == 0)
1237			break;
1238		
1239		while((*p & 0xff) > ' ')
1240		{
1241			fputc(*p++, f_out);
1242		}
1243		fputc(0,f_out);
1244	}
1245	fputc(RULE_GROUP_END,f_out);
1246
1247	return(0);
1248}
1249
1250
1251static int compile_dictrules(FILE *f_in, FILE *f_out, char *fname_temp)
1252{//====================================================================
1253	char *prule;
1254	unsigned char *p;
1255	int ix;
1256	int c;
1257	int gp;
1258	FILE *f_temp;
1259	int n_rules=0;
1260	int count=0;
1261	int different;
1262	const char *prev_rgroup_name;
1263	unsigned int char_code;
1264	int compile_mode=0;
1265	char *buf;
1266	char buf1[200];
1267	char *rules[N_RULES];
1268
1269	int n_rgroups = 0;
1270	RGROUP rgroup[N_RULE_GROUP2];
1271	
1272	linenum = 0;
1273	group_name[0] = 0;
1274
1275	if((f_temp = fopen_log(fname_temp,"wb")) == NULL)
1276		return(1);
1277
1278	for(;;)
1279	{
1280		linenum++;
1281		buf = fgets(buf1,sizeof(buf1),f_in);
1282		if(buf != NULL)
1283		{
1284			if((p = (unsigned char *)strstr(buf,"//")) != NULL)
1285				*p = 0;
1286
1287			if(buf[0] == '\r') buf++;  // ignore extra \r in \r\n 
1288		}
1289
1290		if((buf == NULL) || (buf[0] == '.'))
1291		{
1292			// next .group or end of file, write out the previous group
1293
1294			if(n_rules > 0)
1295			{
1296				strcpy(rgroup[n_rgroups].name,group_name);
1297				rgroup[n_rgroups].start = ftell(f_temp);
1298				output_rule_group(f_temp,n_rules,rules,group_name);
1299				rgroup[n_rgroups].length = ftell(f_temp) - rgroup[n_rgroups].start;
1300				n_rgroups++;
1301
1302				count += n_rules;
1303			}
1304			n_rules = 0;
1305
1306			if(compile_mode == 2)
1307			{
1308				// end of the character replacements section
1309				fwrite(&n_rules,1,4,f_out);   // write a zero word to terminate the replacemenmt list
1310				compile_mode = 0;
1311			}
1312
1313			if(buf == NULL) break;   // end of file
1314
1315			if(memcmp(buf,".L",2)==0)
1316			{
1317				if(compile_lettergroup(&buf[2], f_out) != 0)
1318				{
1319					fprintf(f_log,"%5d: Bad lettergroup\n",linenum);
1320					error_count++;
1321				}
1322				continue;
1323			}
1324
1325			if(memcmp(buf,".replace",8)==0)
1326			{
1327				compile_mode = 2;
1328				fputc(RULE_GROUP_START,f_out);
1329				fputc(RULE_REPLACEMENTS,f_out);
1330
1331				// advance to next word boundary
1332				while((ftell(f_out) & 3) != 0)
1333					fputc(0,f_out);
1334			}
1335
1336			if(memcmp(buf,".group",6)==0)
1337			{
1338				compile_mode = 1;
1339
1340				p = (unsigned char *)&buf[6];
1341				while((p[0]==' ') || (p[0]=='\t')) p++;    // Note: Windows isspace(0xe1) gives TRUE !
1342				ix = 0;
1343				while((*p > ' ') && (ix < LEN_GROUP_NAME))
1344					group_name[ix++] = *p++;
1345				group_name[ix]=0;
1346	
1347				if(sscanf(group_name,"0x%x",&char_code)==1)
1348				{
1349					// group character is given as a character code (max 16 bits)
1350					p = (unsigned char *)group_name;
1351	
1352					if(char_code > 0x100)
1353					{
1354						*p++ = (char_code >> 8);
1355					}
1356					*p++ = char_code;
1357					*p = 0;
1358				}
1359	
1360				if(strlen(group_name) > 2)
1361				{
1362					if(utf8_in(&c,group_name,0) < 2)
1363					{
1364						fprintf(f_log,"%5d: Group name longer than 2 bytes (UTF8)",linenum);
1365						error_count++;
1366					}
1367	
1368					group_name[2] = 0;
1369				}
1370			}
1371
1372			continue;
1373		}
1374		
1375		switch(compile_mode)
1376		{
1377		case 1:    //  .group
1378			prule = compile_rule(buf);
1379			if((prule != NULL) && (n_rules < N_RULES))
1380			{
1381				rules[n_rules++] = prule;
1382			}
1383			break;
1384
1385		case 2:   //  .replace
1386			{
1387				int replace1;
1388				int replace2;
1389				char *p;
1390
1391				p = buf;
1392				replace1 = 0;
1393				replace2 = 0;
1394				while(isspace2(*p)) p++;
1395				ix = 0;
1396				while((unsigned char)(*p) > 0x20)   // not space or zero-byte
1397				{
1398					p += utf8_in(&c,p,0);
1399					replace1 += (c << ix);
1400					ix += 16;
1401				}
1402				while(isspace2(*p)) p++;
1403				ix = 0;
1404				while((unsigned char)(*p) > 0x20)
1405				{
1406					p += utf8_in(&c,p,0);
1407					replace2 += (c << ix);
1408					ix += 16;
1409				}
1410				if(replace1 != 0)
1411				{
1412					Write4Bytes(f_out,replace1);   // write as little-endian
1413					Write4Bytes(f_out,replace2);   // if big-endian, reverse the bytes in LoadDictionary()
1414				}
1415			}
1416			break;
1417		}
1418	}
1419	fclose(f_temp);
1420
1421	qsort((void *)rgroup,n_rgroups,sizeof(rgroup[0]),(int (__cdecl *)(const void *,const void *))rgroup_sorter);
1422
1423	if((f_temp = fopen(fname_temp,"rb"))==NULL)
1424		return(2);
1425
1426	prev_rgroup_name = "\n";
1427
1428	for(gp = 0; gp < n_rgroups; gp++)
1429	{
1430		fseek(f_temp,rgroup[gp].start,SEEK_SET);
1431
1432		if((different = strcmp(rgroup[gp].name, prev_rgroup_name)) != 0)
1433		{
1434			// not the same as the previous group
1435			if(gp > 0)
1436				fputc(RULE_GROUP_END,f_out);
1437			fputc(RULE_GROUP_START,f_out);
1438			fprintf(f_out, prev_rgroup_name = rgroup[gp].name);
1439			fputc(0,f_out);
1440		}
1441
1442		for(ix=rgroup[gp].length; ix>0; ix--)
1443		{
1444			c = fgetc(f_temp);
1445			fputc(c,f_out);
1446		}
1447
1448		if(different)
1449		{
1450		}
1451	}
1452	fputc(RULE_GROUP_END,f_out);
1453	fputc(0,f_out);
1454
1455	fclose(f_temp);
1456	remove(fname_temp);
1457
1458	fprintf(f_log,"\t%d rules, %d groups\n\n",count,n_rgroups);
1459	return(0);
1460}  //  end of compile_dictrules
1461
1462
1463
1464
1465int CompileDictionary(const char *dsource, const char *dict_name, FILE *log, char *fname_err, int flags)
1466{//=====================================================================================================
1467// fname:  space to write the filename in case of error
1468// flags: bit 0:  include source line number information, for debug purposes.
1469
1470	FILE *f_in;
1471	FILE *f_out;
1472	int offset_rules=0;
1473	int value;
1474	char fname_in[sizeof(path_home)+45];
1475	char fname_out[sizeof(path_home)+15];
1476	char fname_temp[sizeof(path_home)+15];
1477	char path[sizeof(path_home)+40];       // path_dsource+20
1478
1479	error_count = 0;
1480	debug_flag = flags & 1;
1481
1482	if(dsource == NULL)
1483		dsource = "";
1484
1485	f_log = log;
1486//f_log = fopen("log2.txt","w");
1487	if(f_log == NULL)
1488		f_log = stderr;
1489
1490	sprintf(path,"%s%s_",dsource,dict_name);
1491	sprintf(fname_in,"%srules",path);
1492	f_in = fopen_log(fname_in,"r");
1493	if(f_in == NULL)
1494	{
1495		if(fname_err)
1496			strcpy(fname_err,fname_in);
1497		return(-1);
1498	}
1499
1500	sprintf(fname_out,"%s%c%s_dict",path_home,PATHSEP,dict_name);
1501	if((f_out = fopen_log(fname_out,"wb+")) == NULL)
1502	{
1503		if(fname_err)
1504			strcpy(fname_err,fname_in);
1505		return(-1);
1506	}
1507	sprintf(fname_temp,"%s%ctemp",path_home,PATHSEP);
1508
1509	transpose_offset = 0;
1510
1511	if(strcmp(dict_name,"ru") == 0)
1512	{
1513		// transpose cyrillic alphabet from unicode to iso8859-5
1514//		transpose_offset = 0x430-0xd0;
1515		transpose_offset = 0x42f;   // range 0x01 to 0x22
1516		transpose_min = 0x430;
1517		transpose_max = 0x451;
1518	}
1519
1520	value = N_HASH_DICT;
1521	Write4Bytes(f_out,value);
1522	Write4Bytes(f_out,offset_rules);
1523
1524	compile_dictlist_start();
1525
1526	fprintf(f_log,"Using phonemetable: '%s'\n",PhonemeTabName());
1527	compile_dictlist_file(path,"roots");
1528	if(translator->langopts.listx)
1529	{
1530		compile_dictlist_file(path,"list");
1531		compile_dictlist_file(path,"listx");
1532	}
1533	else
1534	{
1535		compile_dictlist_file(path,"listx");
1536		compile_dictlist_file(path,"list");
1537	}
1538	compile_dictlist_file(path,"extra");
1539	
1540	compile_dictlist_end(f_out);
1541	offset_rules = ftell(f_out);
1542	
1543	fprintf(f_log,"Compiling: '%s'\n",fname_in);
1544
1545	compile_dictrules(f_in,f_out,fname_temp);
1546	fclose(f_in);
1547
1548	fseek(f_out,4,SEEK_SET);
1549	Write4Bytes(f_out,offset_rules);
1550	fclose(f_out);
1551
1552	translator->LoadDictionary(dict_name,0);
1553
1554	return(error_count);
1555}  //  end of compile_dictionary
1556