/python/engine/PinYin/tools/annotate.py

# -*- coding: utf-8 -*-
import sys
import traceback

dic = {}
for l in file ("../pinyin_table.txt"):
	ll = unicode(l,"utf8").strip("\n\r")
	p = (ll).split(u"\t")
	if p[0] not in dic:
		dic[p[0]] = [(p[1],int(p[2]))]
	else:
		q = dic[p[0]]
		q.append((p[1],int(p[2])))
		q.sort(key = lambda x: x[1], reverse = True)
		if q[-1][1] < 20 * q[-2][1]:
			del q[-1]

dic[u"的"] = [("de",)]
dic[u"地"] = [("de",)]

#~ import re
#~ output = open("duoyin_phrase","w")
#~ ex = re.compile(ur"[a-zA-Z]+|\w",re.UNICODE)
#~ ex_py = re.compile(ur"[a-z\']+",re.UNICODE)
#~ for l in file("QQ_pinyin_ck1.txt"):
	#~ ll = unicode(l[:-1],"utf8").strip("\n\r")
	#~ if ex_py.match(ll):
		#~ py = ll
	#~ else:
		#~ for i in ll:
			#~ if len( dic[i] ) >1:
				#~ print >> output, ll.encode ("utf-8"), py
				#~ break
#~ close(output)

#~ dic_duoyin = {}
#~ output = open("duoyin_phrase","w")
#~ for l in file ("duoyin_phrase_manual.txt"):
	#~ ll = unicode(l,"utf8")[:-1]
	#~ if not ll:
		#~ continue
	#~ p = (ll).split(" ")
	#~ if p[0] in dic_duoyin:
		#~ print >> output, p[0].encode ("utf-8"), dic_duoyin[p[0]]
		#~ print >> output, p[0].encode ("utf-8"), p[1]
	#~ dic_duoyin[p[0]] = p[1]
#~ close(output)
import bz2
dic_duoyin = {}
#~ for l in bz2.BZ2File("duoyin_phrase_manual.txt.bz2","r"):
for l in file("duoyin_phrase_manual.txt"):
	ll = unicode(l,"utf8")[:-1]
	if not ll:
		continue
	p = (ll).split(" ")
	try:
		if p[0] in dic_duoyin:
			dic_duoyin[p[0]].append(p[1])
		else:
			dic_duoyin[p[0]] = [ p[1] ]
	except:
		print >>sys.stderr, "error on load duoyin_phrase_manual"
		sys.exit (1)




#~ import re
#~ ex = re.compile(ur"[a-zA-Z]+|\w",re.UNICODE)
#~ ex_py = re.compile(ur"[a-zA-Z]+",re.UNICODE)
#~ for l in file("qq_pinyin_1.0.txt"):
	#~ ll = unicode(l[:-1],"utf8")
	#~ if 
	#~ p = ex.findall(ll)
	#~ phrase = u""
	#~ py = []
	#~ tag = 0
	#~ while p:
		#~ phrase += p[0]
		#~ if len(p) > 1 and ex_py.match(p[1]):
			#~ py.append(p[1])
			#~ del p[:2]
			#~ tag = 1
		#~ else:
			#~ py.append(dic[p[0]][0][0])
			#~ del p[0]
	#~ if tag:
		#~ print phrase.encode ("utf-8"), "'".join(py)
#~ dic_duoyin = {}



#~ import ZhengJu
#~ editor = ZhengJu.Editor()
#~ i=0
#~ count = 0
#~ for l in file ("google_annotated_uniq.txt"):
	#~ ll = unicode(l,"utf8")
	#~ p = (ll).split(u"\t")
	#~ print p[0]
	#~ if len(p[0])==1:
		#~ count += 1
		#~ i += int(p[2])
	#~ try:
		#~ editor.reverse(p[0])
		#~ pinyin = "'".join (map (str, editor.wordlist))
		#~ print (p[0] + "\t" + pinyin + "\t" + p[1]).encode ("utf-8"),
	#~ except:
		#~ pass
#~ print i,count

#~ for l in file ("google.txt"):
	#~ ll = unicode(l,"utf8")
	#~ p = (ll).split(u"\t")
	#~ print p[0]
	#~ if len(p[0])==1:
		#~ count += 1
		#~ i += int(p[1])
		#~ print p[0].encode ("utf-8")
	#~ try:
		#~ editor.reverse(p[0])
		#~ pinyin = "'".join (map (str, editor.wordlist))
		#~ print (p[0] + "\t" + pinyin + "\t" + p[1]).encode ("utf-8"),
	#~ except:
		#~ pass
#~ print i,count
#~ 
#~ import sets as Set
#~ i=0
#~ count = 0
#~ s = Set.Set()
#~ for l in file ("google_annotated.txt"):
	#~ ll = unicode(l,"utf8")
	#~ p = (ll).split(u"\t")
	#~ if p[0] not in s:
		#~ s.add(p[0])
		#~ print ll.encode ("utf-8"),

traditional = u"\
準陞於術萬與專業叢東絲丟兩嚴喪個臨為麗舉義烏樂喬習鄉書買亂爭虧亙亞產畝親褻億僅從侖\
倉儀們價眾優會傴傘偉傳傷倀倫傖偽佇體僉俠侶僥偵側僑儈儕儂俁儔儼倆儷儉債傾傯僂僨償儻儐儲儺\
兒兌兗黨蘭關興茲養獸囅內岡冊寫軍農馮沖決況凍淨淒涼減湊凜鳳鳧憑凱擊鑿芻劉則剛創刪別剄剎劊劌\
剴劑剮劍剝劇勸辦務勱動勵勁勞勢勛勻匭匱區醫華協單賣盧鹵臥衛卻巹廠廳歷厲壓厭厙廁廂厴廈廚廄\
縣叁參雙發變敘疊葉號嘆嘰嚇呂嗎噸聽啟吳吶嘸囈嘔嚦唄員咼嗆嗚詠嚨嚀響啞噠嘵嗶噦嘩噲嚌噥喲嘜嘮\
嗩喚嘖嗇囀嚙嘯噴嘍嚳囁噯噓嚶囑嚕囂團園囪圍圇國圖圓聖壙場壞塊堅壇壢壩塢墳墜壟壚壘墾堊墊埡塏\
塒塤堝塹墮牆壯聲殼壺處備復夠頭夾奪奩奐奮獎奧妝婦媽嫵嫗媯姍婁婭嬈嬌孌娛媧嫻嬰嬋嬸媼嬡嬪嬙嬤\
孫學孿寧寶實寵審憲宮寬賓寢對尋導壽將爾塵嘗堯尷盡層屜屆屬屢屨嶼歲豈嶇崗峴嵐島嶺巋嶧峽嶠崢巒\
嶗崍嶄嶸崳嶁巔鞏胇幣帥師幃帳幟帶幀幫幬幘幗冪廣慶廬廡庫應廟龐廢廩開異棄弒張彌弳彎彈強歸當錄\
彥徹徑徠憶懺憂愾懷態慫憮慪悵愴憐總懟懌戀恆懇惡慟懨愷惻惱惲悅愨懸慳憫驚懼慘懲憊愜慚憚慣慍憤\
憒懾懣懶懍戇戔戲戧戰戩戶執擴捫掃揚擾撫拋摶摳掄搶護報擔擬攏揀擁攔擰撥擇摯攣撾撻挾撓擋撟掙擠\
揮撈損撿換搗據擄摑擲撣摻摜攬撳攙擱摟攪攜攝攄擺搖擯攤攖撐攆擷擼攛擻攢敵斂數齋斕斬斷無舊時曠\
曇晝顯晉曉曄暈暉暫曖朮機殺雜權條來楊榪極構樅樞棗櫪棖槍楓梟檸檉梔柵標棧櫛櫳棟櫨櫟欄樹棲樣欒\
椏橈楨檔榿橋樺檜槳樁夢檢櫺槨櫝槧欏橢樓欖櫬櫚櫸檻檳櫧橫檣櫻櫫櫥櫓櫞檁歡歟歐殲歿殤殘殞殮殫殯\
毆毀轂畢斃氈毿氌氣氫氬氳汆匯漢湯洶溝沒灃漚瀝淪滄溈滬濘淚澩瀧瀘濼瀉潑澤涇潔灑窪浹淺漿澆湞濁\
測澮濟瀏渾滸濃潯濤澇淶漣潿渦渙滌潤澗漲澀澱淵淥漬瀆漸澠漁瀋滲溫灣濕潰濺漵潷滾滯灩灄滿瀅濾濫\
灤濱灘瀠瀟瀲濰潛瀦瀾瀨瀕灝滅燈靈災燦煬爐燉煒熗點煉熾爍爛烴燭煙煩燒燁燴燙燼熱煥燜燾愛爺牘牽\
犧犢狀獷猶狽獰獨狹獅獪猙獄猻獫獵獼玀豬貓獻獺璣瑪瑋環現璽玨琺瓏琿璉瑣瓊瑤璦瓔瓚甌電畫暢畬疇\
癤療瘧癘瘍瘡瘋癰痙癢癆瘓癇痺癉瘞癟癱癮癭癩癬癲皚皺皸盞鹽監蓋盜盤瞇著睜睞瞼睪瞞矚矯磯礬礦碭\
碼磚硨硯礪礱礫礎碩硤磽確鹼礙磧磣鹼礡禮檷禎禱禍稟祿禪離禿稈種祕積稱穢稅穌穩穡窮竊竅窯竄窩窺\
竇窶豎競篤筍筆筧箋籠籩篳篩箏籌簽簡簀篋籜籮簞簫簣簍籃籬籪籟糴類糶糲粵糞糧糝緊縶糾紆紅紂纖紇\
約級紈纊紀紉緯紜純紕紗綱納縱綸紛紙紋紡紐紓線紺紲紱練組紳細織終縐絆紼絀紹繹經紿綁絨結繞絎繪\
給絢絳絡絕絞統綆綃絹繡綏絛繼綈績緒綾續綺緋綽緄繩維綿綬繃綢綹綣綜綻綰綠綴緇緙緗緘緬纜緹緲緝\
繢緦綞緞緶緱縋緩締縷編緡緣縉縛縟縝縫縞纏縭縊縑繽縹縵縲纓縮繆繅纈繚繕繒繾繰繯繳纘罌網羅罰罷\
羆羈羥羨翹耬聳恥聶聾職聹聯聵聰肅腸膚骯腎腫脹脅冑膽勝朧臚脛膠脈膾臟臍腦膿臠腳脫腡臉臘膩膃騰\
臏輿艤艦艙艫艱艷藝節羋薌蕪蘆蓯葦藶莧萇蒼苧蘇蘋莖蘢蔦塋煢繭荊莢蕘蓽蕎薈薺蕩榮葷滎犖熒蕁藎蓀\
蔭葒藥蒞萊蓮蒔萵薟獲蕕瑩鶯蘿螢營縈蕭薩蔥蕆蕢蔣蔞藍薊蘺蕷鎣驀薔蘞藺藹蘄蘊藪蘚虜慮虛虯雖蝦蠆\
蝕蟻螞蠶蜆蠱蠣蟶蠻蟄蛺蟯螄蠐蛻蝸蠟蠅蟈蟬蠍螻蠑蠷釁銜補襯袞襖裊襪襲裝襠褳襝褲襉褸襤見觀規覓\
視覘覽覺覬覡覿覦覯覲覷觴觸觶譽謄計訂訃認譏訐訌討讓訕訖訓議訊記講諱謳詎訝訥許訛論訟諷設訪訣\
証詁訶評詛識詐訴診詆謅詞詘詔譯詒誆誄試詿詩詰詼誠誅詵話誕詬詮詭詢詣諍該詳詫諢詡誡誣語誚誤誥\
誘誨誑說誦誒請諸諏諾讀諑誹課諉諛誰諗調諂諒諄誶談誼謀諶諜謊諫諧謔謁謂諤諭諼讒諮諳諺諦謎諞謨\
讜謖謝謠謗謚謙謐謹謾謫譾謬譚譖譙讕譜譎讞譴譫讖貝貞負貢財責賢敗賬貨質販貪貧貶購貯貫貳賤賁貰\
貼貴貺貸貿費賀貽賊贄賈賄貲賃賂贓資賅贐賕賑賚賒賦賭贖賞賜賡賠賧賴贅賻賺賽賾贗贊贈贍贏贛趙趕\
趨趲躉躍蹌躒踐蹺蹕躚躋踴躊蹤躓躑躡蹣躕躥躪躦軀車軋軌軒軔轉軛輪軟轟軻轤軸軹軼軫轢軺輕軾載輊\
轎輇輅較輒輔輛輦輩輝輥輞輟輜輳輻輯輸轡轅轄輾轆轍轔辭辯辮邊遼達遷過邁運還這進遠違連遲邇逕跡\
適選遜遞邐邏遺遙鄧鄺鄔郵鄒鄴鄰郟鄶鄭鄆酈鄖鄲醞醬釅釃釀釋鑑鑾鏨釓釔針釘釗釙釕釷釧釤釩釣鍆釹\
釵鈣鈦鉅鈍鈔鐘鈉鋇鋼鈑鈐鑰欽鈞鎢鉤鈧鈁鈥鈄鈕鈀鈺錢鉦鉗鈷缽鈳鈽鈸鉞鑽鉬鉭鉀鈿鈾鐵鉑鈴鑠鉛鉚\
鈰鉉鉈鉍鈮鈹鐸銬銠鉺銪鋮鋏鐃鐺銅鋁銦鎧鍘銖銑鋌銩鏵銓鎩鉿銚鉻銘錚銫鉸銥鏟銃鐋銨銀銣鑄鐒鋪錸\
鋱鏈鏗銷鎖鋰鋤鍋鋯鋨鏽銼鋝鋒鋅銳銻鋃鋟鋦錒錆鍺錯錨錛錁錕錫錮鑼錘錐錦錈錟錠鍵鋸錳錙鍥鍇鏘鍶\
鍔鍤鍬鍾鍛鎪鍰鍍鎂鏤鐨鏌鎮鎘鑷鐫鎳鎦鎬鎊鎰鎵鑌鎔鏢鏜鏝鏍鏞鏡鏑鏃鏇鐔鐐鏷鐓鑭鐠鏹鐙鑊鐳鐲鐮\
鐿鑣鑲長門閂閃閆閉問闖閏闈閑閎間閔閌悶閘鬧閨聞闥閩閭閥閣閡閫鬮閱閬閾閹閶鬩閿閽閻閼闡闌闃闊\
闋闔闐闕闞隊陽陰陣階際陸隴陳陘陝隉隕險隨隱隸雋難雛讎靂霧霽靄靚靜靨韃韉韋韌韓韙韞韜韻頁頂頃\
頇項順須頊頑顧頓頎頒頌頏預顱領頗頸頡頰頜潁頦頤頻頹頷穎顆題顎顓顏額顳顢顛顙顥顫顰顴風颮颯颶\
颼飄飆飛饗饜飢餳飩餼飪飫飭飯飲餞飾飽飼飴餌饒餉餃餅餑餓餘餒餛餡館饋餿饞饃餾饈饉饅饌馬馭馱馴\
馳驅駁驢駔駛駟駙駒騶駐駝駑駕驛駘驍罵驕驊駱駭駢驪騁驗駿騏騎騍騅驂騙騭騷騖驁騮騫騸驃騾驄驏驟\
驥驤髏髖髕鬢魘魎魚魷魯魴鱍鱸鮒鮑鱟鮐鮭鮚鮪鮞鱭鮫鮮鯗鱘鯁鱺鰱鰹鯉鰣鰷鯀鯊鯇鯽鯖鯪鯫鯡鯤鯧鯢\
鯰鯛鯨鯔鰈鰓鱷鰍鰒鰉鰲鰭鰨鰥鰩鰳鰾鱈鱉鰻鱖鱔鱗鱒鱧鳥鳩雞鳶鳴鷗鴉鴇鴆鴣鶇鸕鴨鴦鴟鴝鴛鴕鷥鷙\
鴯鴰鵂鴿鸞鴻鵓鸝鵑鵠鵝鵒鷴鵜鵡鵲鵪鵯鵬鶉鶘鶚鶻鶩鷂鶼鶴鸚鷓鷚鷯鷦鷲鷸鷺鷹鸛鹺麥麩麼黃黌黷黲\
黽黿鼉鼴齊齏齒齔齟齡齙齠齜齦齬齪齲齷龍龔龕龜"
def filter(phrase):
	for i in phrase:
		if i in traditional:
			return False
	else:
		return True
def annotate(phrase):
	if len(phrase) == 0:
		yield ""
	elif len(phrase) == 1:
		for i in dic[phrase]:
			yield i[0]
	else:
		for i in range(len(phrase),1, -1):
			if phrase[:i] in dic_duoyin:
				for q in dic_duoyin[phrase[:i]]:
					for p in annotate(phrase[i:]):
						yield q + "'" +  p
				break
		else:
			for i in dic[phrase[0]]:
				for q in annotate(phrase[1:]):
					yield i[0] + "'" +  q

def annotate_single(phrase):
	if len(phrase) == 0:
		yield ""
	elif len(phrase) == 1:
		yield dic[phrase][0]
	else:
		for i in range(len(phrase),1, -1):
			if phrase[:i] in dic_duoyin:
				for q in dic_duoyin[phrase[:i]]:
					for p in annotate(phrase[i:]):
						yield q + "'" +  p
				break
		else:
			for q in annotate(phrase[1:]):
				yield dic[phrase[0]][0][0] + "'" +  q
out_put = open("phrase_freq_annotated.txt", "w")
out_put2 = open("phrase_freq_ambiguous.txt", "w")
out_put3 = open("phrase_freq_guessed.txt", "w")
for l in file("phrase_freq.txt"):
	ll = unicode(l[:-1],"utf8")
	p = (ll).split(u"\t")
	if not filter(p[0]):
		continue
	if int(p[1]) < 10000:
		continue
	try:
		py = list(annotate(p[0]))
		if len(py) == 1:
			print >> out_put, u"\t".join ([p[0], py[0], p[1]]).encode ("utf8")
		else:
			for i in annotate_single(p[0]):
				print >> out_put3, u"\t".join ([p[0], i, p[1]]).encode ("utf8")
			for i in py:
				print >> out_put2, u"\t".join ([p[0], i, p[1]]).encode ("utf8")
	except KeyboardInterrupt, e:
		print >> sys.stderr, "Exit"
		sys.exit (1)
	except:
		print >>sys.stderr, p[0].encode("utf-8")
		traceback.print_exc()