/ime/latinime/src/com/googlecode/eyesfree/inputmethod/latin/BinaryDictionary.java

  1/*
  2 * Copyright (C) 2008 The Android Open Source Project
  3 * 
  4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  5 * use this file except in compliance with the License. You may obtain a copy of
  6 * the License at
  7 * 
  8 * http://www.apache.org/licenses/LICENSE-2.0
  9 * 
 10 * Unless required by applicable law or agreed to in writing, software
 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 13 * License for the specific language governing permissions and limitations under
 14 * the License.
 15 */
 16
 17package com.googlecode.eyesfree.inputmethod.latin;
 18
 19import android.content.Context;
 20import android.util.Log;
 21
 22import java.io.IOException;
 23import java.io.InputStream;
 24import java.nio.ByteBuffer;
 25import java.nio.ByteOrder;
 26import java.nio.channels.Channels;
 27import java.util.Arrays;
 28
 29/**
 30 * Implements a static, compacted, binary dictionary of standard words.
 31 */
 32public class BinaryDictionary extends Dictionary {
 33
 34    /**
 35     * There is difference between what java and native code can handle.
 36     * This value should only be used in BinaryDictionary.java
 37     * It is necessary to keep it at this value because some languages e.g. German have
 38     * really long words.
 39     */
 40    protected static final int MAX_WORD_LENGTH = 48;
 41
 42    private static final String TAG = "BinaryDictionary";
 43    private static final int MAX_ALTERNATIVES = 16;
 44    private static final int MAX_WORDS = 18;
 45    private static final int MAX_BIGRAMS = 60;
 46
 47    private static final int TYPED_LETTER_MULTIPLIER = 2;
 48    private static final boolean ENABLE_MISSED_CHARACTERS = true;
 49
 50    private int mDicTypeId;
 51    private int mNativeDict;
 52    private int mDictLength;
 53    private int[] mInputCodes = new int[MAX_WORD_LENGTH * MAX_ALTERNATIVES];
 54    private char[] mOutputChars = new char[MAX_WORD_LENGTH * MAX_WORDS];
 55    private char[] mOutputChars_bigrams = new char[MAX_WORD_LENGTH * MAX_BIGRAMS];
 56    private int[] mFrequencies = new int[MAX_WORDS];
 57    private int[] mFrequencies_bigrams = new int[MAX_BIGRAMS];
 58    // Keep a reference to the native dict direct buffer in Java to avoid
 59    // unexpected deallocation of the direct buffer.
 60    private ByteBuffer mNativeDictDirectBuffer;
 61
 62    static {
 63        try {
 64            System.loadLibrary("jni_touchlatinime");
 65        } catch (UnsatisfiedLinkError ule) {
 66            Log.e("BinaryDictionary", "Could not load native library jni_latinime");
 67        }
 68    }
 69
 70    /**
 71     * Create a dictionary from a raw resource file
 72     * @param context application context for reading resources
 73     * @param resId the resource containing the raw binary dictionary
 74     */
 75    public BinaryDictionary(Context context, int[] resId, int dicTypeId) {
 76        if (resId != null && resId.length > 0 && resId[0] != 0) {
 77            loadDictionary(context, resId);
 78        }
 79        mDicTypeId = dicTypeId;
 80    }
 81
 82    /**
 83     * Create a dictionary from a byte buffer. This is used for testing.
 84     * @param context application context for reading resources
 85     * @param byteBuffer a ByteBuffer containing the binary dictionary
 86     */
 87    public BinaryDictionary(Context context, ByteBuffer byteBuffer, int dicTypeId) {
 88        if (byteBuffer != null) {
 89            if (byteBuffer.isDirect()) {
 90                mNativeDictDirectBuffer = byteBuffer;
 91            } else {
 92                mNativeDictDirectBuffer = ByteBuffer.allocateDirect(byteBuffer.capacity());
 93                byteBuffer.rewind();
 94                mNativeDictDirectBuffer.put(byteBuffer);
 95            }
 96            mDictLength = byteBuffer.capacity();
 97            mNativeDict = openNative(mNativeDictDirectBuffer,
 98                    TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER);
 99        }
100        mDicTypeId = dicTypeId;
101    }
102
103    private native int openNative(ByteBuffer bb, int typedLetterMultiplier,
104            int fullWordMultiplier);
105    private native void closeNative(int dict);
106    private native boolean isValidWordNative(int nativeData, char[] word, int wordLength);
107    private native int getSuggestionsNative(int dict, int[] inputCodes, int codesSize, 
108            char[] outputChars, int[] frequencies, int maxWordLength, int maxWords,
109            int maxAlternatives, int skipPos, int[] nextLettersFrequencies, int nextLettersSize);
110    private native int getBigramsNative(int dict, char[] prevWord, int prevWordLength,
111            int[] inputCodes, int inputCodesLength, char[] outputChars, int[] frequencies,
112            int maxWordLength, int maxBigrams, int maxAlternatives);
113
114    private final void loadDictionary(Context context, int[] resId) {
115        InputStream[] is = null;
116        try {
117            // merging separated dictionary into one if dictionary is separated
118            int total = 0;
119            is = new InputStream[resId.length];
120            for (int i = 0; i < resId.length; i++) {
121                is[i] = context.getResources().openRawResource(resId[i]);
122                total += is[i].available();
123            }
124
125            mNativeDictDirectBuffer =
126                ByteBuffer.allocateDirect(total).order(ByteOrder.nativeOrder());
127            int got = 0;
128            for (int i = 0; i < resId.length; i++) {
129                 got += Channels.newChannel(is[i]).read(mNativeDictDirectBuffer);
130            }
131            if (got != total) {
132                Log.e(TAG, "Read " + got + " bytes, expected " + total);
133            } else {
134                mNativeDict = openNative(mNativeDictDirectBuffer,
135                        TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER);
136                mDictLength = total;
137            }
138        } catch (IOException e) {
139            Log.w(TAG, "No available memory for binary dictionary");
140        } finally {
141            try {
142                if (is != null) {
143                    for (int i = 0; i < is.length; i++) {
144                        is[i].close();
145                    }
146                }
147            } catch (IOException e) {
148                Log.w(TAG, "Failed to close input stream");
149            }
150        }
151    }
152
153
154    @Override
155    public void getBigrams(final WordComposer codes, final CharSequence previousWord,
156            final WordCallback callback, int[] nextLettersFrequencies) {
157
158        char[] chars = previousWord.toString().toCharArray();
159        Arrays.fill(mOutputChars_bigrams, (char) 0);
160        Arrays.fill(mFrequencies_bigrams, 0);
161
162        int codesSize = codes.size();
163        Arrays.fill(mInputCodes, -1);
164        int[] alternatives = codes.getCodesAt(0);
165        System.arraycopy(alternatives, 0, mInputCodes, 0,
166                Math.min(alternatives.length, MAX_ALTERNATIVES));
167
168        int count = getBigramsNative(mNativeDict, chars, chars.length, mInputCodes, codesSize,
169                mOutputChars_bigrams, mFrequencies_bigrams, MAX_WORD_LENGTH, MAX_BIGRAMS,
170                MAX_ALTERNATIVES);
171
172        for (int j = 0; j < count; j++) {
173            if (mFrequencies_bigrams[j] < 1) break;
174            int start = j * MAX_WORD_LENGTH;
175            int len = 0;
176            while (mOutputChars_bigrams[start + len] != 0) {
177                len++;
178            }
179            if (len > 0) {
180                callback.addWord(mOutputChars_bigrams, start, len, mFrequencies_bigrams[j],
181                        mDicTypeId, DataType.BIGRAM);
182            }
183        }
184    }
185
186    @Override
187    public void getWords(final WordComposer codes, final WordCallback callback,
188            int[] nextLettersFrequencies) {
189        final int codesSize = codes.size();
190        // Won't deal with really long words.
191        if (codesSize > MAX_WORD_LENGTH - 1) return;
192        
193        Arrays.fill(mInputCodes, -1);
194        for (int i = 0; i < codesSize; i++) {
195            int[] alternatives = codes.getCodesAt(i);
196            System.arraycopy(alternatives, 0, mInputCodes, i * MAX_ALTERNATIVES,
197                    Math.min(alternatives.length, MAX_ALTERNATIVES));
198        }
199        Arrays.fill(mOutputChars, (char) 0);
200        Arrays.fill(mFrequencies, 0);
201
202        int count = getSuggestionsNative(mNativeDict, mInputCodes, codesSize,
203                mOutputChars, mFrequencies,
204                MAX_WORD_LENGTH, MAX_WORDS, MAX_ALTERNATIVES, -1,
205                nextLettersFrequencies,
206                nextLettersFrequencies != null ? nextLettersFrequencies.length : 0);
207
208        // If there aren't sufficient suggestions, search for words by allowing wild cards at
209        // the different character positions. This feature is not ready for prime-time as we need
210        // to figure out the best ranking for such words compared to proximity corrections and
211        // completions.
212        if (ENABLE_MISSED_CHARACTERS && count < 5) {
213            for (int skip = 0; skip < codesSize; skip++) {
214                int tempCount = getSuggestionsNative(mNativeDict, mInputCodes, codesSize,
215                        mOutputChars, mFrequencies,
216                        MAX_WORD_LENGTH, MAX_WORDS, MAX_ALTERNATIVES, skip,
217                        null, 0);
218                count = Math.max(count, tempCount);
219                if (tempCount > 0) break;
220            }
221        }
222
223        for (int j = 0; j < count; j++) {
224            if (mFrequencies[j] < 1) break;
225            int start = j * MAX_WORD_LENGTH;
226            int len = 0;
227            while (mOutputChars[start + len] != 0) {
228                len++;
229            }
230            if (len > 0) {
231                callback.addWord(mOutputChars, start, len, mFrequencies[j], mDicTypeId,
232                        DataType.UNIGRAM);
233            }
234        }
235    }
236
237    @Override
238    public boolean isValidWord(CharSequence word) {
239        if (word == null) return false;
240        char[] chars = word.toString().toCharArray();
241        return isValidWordNative(mNativeDict, chars, chars.length);
242    }
243
244    public int getSize() {
245        return mDictLength; // This value is initialized on the call to openNative()
246    }
247
248    @Override
249    public synchronized void close() {
250        if (mNativeDict != 0) {
251            closeNative(mNativeDict);
252            mNativeDict = 0;
253        }
254    }
255
256    @Override
257    protected void finalize() throws Throwable {
258        close();
259        super.finalize();
260    }
261}