PageRenderTime 45ms CodeModel.GetById 9ms app.highlight 29ms RepoModel.GetById 1ms app.codeStats 1ms

/extensions/universalchardet/src/base/CharDistribution.h

http://github.com/zpao/v8monkey
C Header | 242 lines | 128 code | 33 blank | 81 comment | 17 complexity | f83343e3957d8ccc0d8855aed181a0d5 MD5 | raw file
  1/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
  2/* ***** BEGIN LICENSE BLOCK *****
  3 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  4 *
  5 * The contents of this file are subject to the Mozilla Public License Version
  6 * 1.1 (the "License"); you may not use this file except in compliance with
  7 * the License. You may obtain a copy of the License at
  8 * http://www.mozilla.org/MPL/
  9 *
 10 * Software distributed under the License is distributed on an "AS IS" basis,
 11 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 12 * for the specific language governing rights and limitations under the
 13 * License.
 14 *
 15 * The Original Code is Mozilla Communicator client code.
 16 *
 17 * The Initial Developer of the Original Code is
 18 * Netscape Communications Corporation.
 19 * Portions created by the Initial Developer are Copyright (C) 1998
 20 * the Initial Developer. All Rights Reserved.
 21 *
 22 * Contributor(s):
 23 *
 24 * Alternatively, the contents of this file may be used under the terms of
 25 * either the GNU General Public License Version 2 or later (the "GPL"), or
 26 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 27 * in which case the provisions of the GPL or the LGPL are applicable instead
 28 * of those above. If you wish to allow use of your version of this file only
 29 * under the terms of either the GPL or the LGPL, and not to allow others to
 30 * use your version of this file under the terms of the MPL, indicate your
 31 * decision by deleting the provisions above and replace them with the notice
 32 * and other provisions required by the GPL or the LGPL. If you do not delete
 33 * the provisions above, a recipient may use your version of this file under
 34 * the terms of any one of the MPL, the GPL or the LGPL.
 35 *
 36 * ***** END LICENSE BLOCK ***** */
 37
 38#ifndef CharDistribution_h__
 39#define CharDistribution_h__
 40
 41#include "nscore.h"
 42
 43#define ENOUGH_DATA_THRESHOLD 1024
 44 
 45#define MINIMUM_DATA_THRESHOLD  4
 46
 47class CharDistributionAnalysis
 48{
 49public:
 50  CharDistributionAnalysis() {Reset(false);}
 51
 52  //feed a block of data and do distribution analysis
 53  void HandleData(const char* aBuf, PRUint32 aLen) {}
 54  
 55  //Feed a character with known length
 56  void HandleOneChar(const char* aStr, PRUint32 aCharLen)
 57  {
 58    PRInt32 order;
 59
 60    //we only care about 2-bytes character in our distribution analysis
 61    order = (aCharLen == 2) ? GetOrder(aStr) : -1;
 62
 63    if (order >= 0)
 64    {
 65      mTotalChars++;
 66      //order is valid
 67      if ((PRUint32)order < mTableSize)
 68      {
 69        if (512 > mCharToFreqOrder[order])
 70          mFreqChars++;
 71      }
 72    }
 73  }
 74
 75  //return confidence base on existing data
 76  float GetConfidence(void);
 77
 78  //Reset analyser, clear any state 
 79  void      Reset(bool aIsPreferredLanguage) 
 80  {
 81    mDone = false;
 82    mTotalChars = 0;
 83    mFreqChars = 0;
 84    mDataThreshold = aIsPreferredLanguage ? 0 : MINIMUM_DATA_THRESHOLD;
 85  }
 86
 87  //This function is for future extension. Caller can use this function to control
 88  //analyser's behavior
 89  void      SetOpion(){}
 90
 91  //It is not necessary to receive all data to draw conclusion. For charset detection,
 92  // certain amount of data is enough
 93  bool GotEnoughData() {return mTotalChars > ENOUGH_DATA_THRESHOLD;}
 94
 95protected:
 96  //we do not handle character base on its original encoding string, but 
 97  //convert this encoding string to a number, here called order.
 98  //This allow multiple encoding of a language to share one frequency table 
 99  virtual PRInt32 GetOrder(const char* str) {return -1;}
100  
101  //If this flag is set to true, detection is done and conclusion has been made
102  bool     mDone;
103
104  //The number of characters whose frequency order is less than 512
105  PRUint32 mFreqChars;
106
107  //Total character encounted.
108  PRUint32 mTotalChars;
109
110  //Number of hi-byte characters needed to trigger detection
111  PRUint32 mDataThreshold;
112
113  //Mapping table to get frequency order from char order (get from GetOrder())
114  const PRInt16  *mCharToFreqOrder;
115
116  //Size of above table
117  PRUint32 mTableSize;
118
119  //This is a constant value varies from language to language, it is used in 
120  //calculating confidence. See my paper for further detail.
121  float    mTypicalDistributionRatio;
122};
123
124
125class EUCTWDistributionAnalysis: public CharDistributionAnalysis
126{
127public:
128  EUCTWDistributionAnalysis();
129protected:
130
131  //for euc-TW encoding, we are interested 
132  //  first  byte range: 0xc4 -- 0xfe
133  //  second byte range: 0xa1 -- 0xfe
134  //no validation needed here. State machine has done that
135  PRInt32 GetOrder(const char* str) 
136  { if ((unsigned char)*str >= (unsigned char)0xc4)  
137      return 94*((unsigned char)str[0]-(unsigned char)0xc4) + (unsigned char)str[1] - (unsigned char)0xa1;
138    else
139      return -1;
140  }
141};
142
143
144class EUCKRDistributionAnalysis : public CharDistributionAnalysis
145{
146public:
147  EUCKRDistributionAnalysis();
148protected:
149  //for euc-KR encoding, we are interested 
150  //  first  byte range: 0xb0 -- 0xfe
151  //  second byte range: 0xa1 -- 0xfe
152  //no validation needed here. State machine has done that
153  PRInt32 GetOrder(const char* str) 
154  { if ((unsigned char)*str >= (unsigned char)0xb0)  
155      return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
156    else
157      return -1;
158  }
159};
160
161class GB2312DistributionAnalysis : public CharDistributionAnalysis
162{
163public:
164  GB2312DistributionAnalysis();
165protected:
166  //for GB2312 encoding, we are interested 
167  //  first  byte range: 0xb0 -- 0xfe
168  //  second byte range: 0xa1 -- 0xfe
169  //no validation needed here. State machine has done that
170  PRInt32 GetOrder(const char* str) 
171  { if ((unsigned char)*str >= (unsigned char)0xb0 && (unsigned char)str[1] >= (unsigned char)0xa1)  
172      return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
173    else
174      return -1;
175  }
176};
177
178
179class Big5DistributionAnalysis : public CharDistributionAnalysis
180{
181public:
182  Big5DistributionAnalysis();
183protected:
184  //for big5 encoding, we are interested 
185  //  first  byte range: 0xa4 -- 0xfe
186  //  second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
187  //no validation needed here. State machine has done that
188  PRInt32 GetOrder(const char* str) 
189  { if ((unsigned char)*str >= (unsigned char)0xa4)  
190      if ((unsigned char)str[1] >= (unsigned char)0xa1)
191        return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0xa1 +63;
192      else
193        return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0x40;
194    else
195      return -1;
196  }
197};
198
199class SJISDistributionAnalysis : public CharDistributionAnalysis
200{
201public:
202  SJISDistributionAnalysis();
203protected:
204  //for sjis encoding, we are interested 
205  //  first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
206  //  second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
207  //no validation needed here. State machine has done that
208  PRInt32 GetOrder(const char* str) 
209  { 
210    PRInt32 order;
211    if ((unsigned char)*str >= (unsigned char)0x81 && (unsigned char)*str <= (unsigned char)0x9f)  
212      order = 188 * ((unsigned char)str[0]-(unsigned char)0x81);
213    else if ((unsigned char)*str >= (unsigned char)0xe0 && (unsigned char)*str <= (unsigned char)0xef)  
214      order = 188 * ((unsigned char)str[0]-(unsigned char)0xe0 + 31);
215    else
216      return -1;
217    order += (unsigned char)*(str+1) - 0x40;
218    if ((unsigned char)str[1] > (unsigned char)0x7f)
219      order--;
220    return order;
221  }
222};
223
224class EUCJPDistributionAnalysis : public CharDistributionAnalysis
225{
226public:
227  EUCJPDistributionAnalysis();
228protected:
229  //for euc-JP encoding, we are interested 
230  //  first  byte range: 0xa0 -- 0xfe
231  //  second byte range: 0xa1 -- 0xfe
232  //no validation needed here. State machine has done that
233  PRInt32 GetOrder(const char* str) 
234  { if ((unsigned char)*str >= (unsigned char)0xa0)  
235      return 94*((unsigned char)str[0]-(unsigned char)0xa1) + (unsigned char)str[1] - (unsigned char)0xa1;
236    else
237      return -1;
238  }
239};
240
241#endif //CharDistribution_h__
242