1 package com.boyter.SpellingCorrector;

2

3 import java.util.*;

9 * user got that correct.

10 */

11 public class SpellingCorrector implements ISpellingCorrector {

12

13 // word to count map - how may times a word is present - or a weight attached to a word

14 private Map<String, Integer> dictionary = null;

15

16 public SpellingCorrector(int lruCount) {

17 this.dictionary = Collections.synchronizedMap(new LruCache<>(lruCount));

18 }

120

121 /**

122 * Return a list of strings which are words similar to our one which could potentially be misspellings

123 * Abuse the fact that a char can be used as an integer

124 * Assume that they got the first letter correct for all edits to cut on CPU burn time

105 }

106

107 public void testSpellingCorrectorExample() {

108 Map<Integer, Set<String>> expectedResultMap = new HashMap<Integer, Set<String>>();

109 expectedResultMap.put(0, new HashSet<String>(Arrays.asList(

126 )));

127

128 String file = "./examples/spellingcorrector.rpl";

129 try {

130 System.out.print("Running test " + file + "... ");

142 }

143

144 public void testSpellingCorrectorExample10K() {

145

146 // We test a couple of different inputs and compare the first match

43 @Component(OperationType.SPELLING_CHECKER)

44 @ResourceMetaData(name = "Simple Spelling Corrector")

45 @Parameters(

46 exclude = {

47 NorvigSpellingCorrector.PARAM_MODEL_LOCATION })

48 @TypeCapability(

49 inputs = {

62 private String modelLocation;

63

64 private NorvigSpellingAlgorithm spellingCorrector;

65

66 @Override

70 super.initialize(context);

71 try {

72 spellingCorrector = new NorvigSpellingAlgorithm();

73 spellingCorrector.train(getContext().getResourceURL(modelLocation), "UTF-8");

19 @PersistJobDataAfterExecution

20 @DisallowConcurrentExecution

21 public class PopulateSpellingCorrectorJob implements Job {

22

23 private final LoggerWrapper logger;

24 public int MAXFILELINEDEPTH = Singleton.getHelpers().tryParseInt(Properties.getProperties().getProperty(Values.MAXFILELINEDEPTH, Values.DEFAULTMAXFILELINEDEPTH), Values.DEFAULTMAXFILELINEDEPTH);

25

26 public PopulateSpellingCorrectorJob() {

27 this.logger = Singleton.getLogger();

28 }

31 Thread.currentThread().setPriority(Thread.MIN_PRIORITY);

32 Path path = Paths.get(Properties.getProperties().getProperty(Values.REPOSITORYLOCATION, Values.DEFAULTREPOSITORYLOCATION));

33 this.logger.info(String.format("4f5b6cb6::starting populatespellingcorrector in path %s", path.toString()));

34

35 try {

13

14 public void testEmptyNullTerms() {

15 ISpellingCorrector sc = this.getSpellingCorrector();

16 assertNull(sc.correct(null));

17 assertEquals("", sc.correct(""));

73 public void testSpellingCorrectorTwoMatchesSameLengthWins() {

74 ISpellingCorrector sc = this.getSpellingCorrector();

75 sc.putWord("test");

76 sc.putWord("tests");

104 */

105 public void testLongStringPerformance() {

106 ISpellingCorrector sc = this.getSpellingCorrector();

107 sc.correct("thisisareallylongstringthatshouldcalusethingstorunreallyslow");

108 }

110 public void testFuzzSpellingCorrector() {

111 Random rand = new Random();

112 ISpellingCorrector sc = getSpellingCorrector();

113

114 for (int j = 0; j < 1000; j++) {

7 public class SpellingCorrectorTest {

8 SpellingCorrector spellingCorrector = new SpellingCorrector();

9

10 @Test

15 words.add("th");

16

17 assertEquals(words, spellingCorrector.applySingleCharacterDeletions("the"));

18 }

19

25 words.add("tets");

26

27 assertEquals(words, spellingCorrector.applyTranspositions("test"));

28 }

29

41 @Test

42 public void shouldReturnTheCorrectWord() throws Exception {

43 assertEquals("access", spellingCorrector.correct("acess"));

44 assertEquals("access", spellingCorrector.correct("access"));

23 * since we can usually assume that the user got that correct.

24 */

25 public class SearchcodeSpellingCorrector implements ISpellingCorrector {

26

27 // How many terms to keep in the LRUCACHE

28 private int LRUCOUNT = Integer.parseInt(Values.DEFAULTSPELLINGCORRECTORSIZE);

29

30 private int VARIATIONSCOUNT = 200000;

35 public SearchcodeSpellingCorrector() {

36 this.LRUCOUNT = Integer.parseInt(Properties.getProperties().getProperty(Values.SPELLINGCORRECTORSIZE, Values.DEFAULTSPELLINGCORRECTORSIZE));

37 if (this.LRUCOUNT <= 0) {

38 this.LRUCOUNT = Integer.parseInt(Values.DEFAULTSPELLINGCORRECTORSIZE);

171

172 /**

173 * Return a list of strings which are words similar to our one which could potentially be misspellings

174 * Abuse the fact that a char can be used as an integer

175 * Assume that they got the first letter correct for all edits to cut on CPU burn time

3 import java.io.IOException;

4

5 import spell.SpellCorrector.NoSimilarWordFoundException;

6

7 /**

8 * A simple main class for running the spelling corrector

9 */

10 public class Main {

22 * Create an instance of your corrector here

23 */

24 SpellCorrector corrector = new SpellingCorrector();

25 Trie trie1 = new Words();

26

13 //If I can get my act together I'll try and refactor it until it's as concise as Norvig's

14 //python implementation

15 public class SpellingCorrector {

16 private static final String alphabet = "abcdefghijklmnopqrstuvwxyz";

17 private Map<String,Integer> languageModel = new HashMap<String,Integer>();

18

19 public SpellingCorrector(){

20 buildLanguageModel();

21 }

3 import java.io.IOException;

4

5 import spell.ISpellCorrector.NoSimilarWordFoundException;

6

7 /**

8 * A simple main class for running the spelling corrector. This class is not

9 * used by the passoff program.

10 */

23 * Create an instance of your corrector here

24 */

25 ISpellCorrector corrector = new Spell();

26

27 corrector.useDictionary(dictionaryFileName);

4 import java.util.ArrayList;

5

6 import qualcheck.SpellCorrector.NoSimilarWordFoundException;

7

8 /**

9 * A simple main class for running the spelling corrector

10 */

11 public class RunQualCheck {

12

13 private Spell corrector;

14

15 public RunQualCheck() {

12 import java.util.*;

13

14 public class SpellingCorrector implements SpellCorrector {

15

16 // @Test

17 // public void test1() throws IOException, NoSimilarWordFoundException {

18 // SpellingCorrector testSpell = new SpellingCorrector();

19 // testSpell.useDictionary("dictionary.txt");

20 // System.out.println(test.toString());

1 package spelling.corrector;

2

3 import java.io.File;

8 import spelling.dictionary.Dictionary;

9

10 public abstract class SpellingCorrectorTest extends TestCase {

11

12 public final void testCorrect(){

13 try {

14 Dictionary.load(Locale.ENGLISH, new File("holmes.txt"));

15 SpellingCorrector sp = getSpellingCorrector();

16 String correction = sp.correct("speling");

17 assertEquals("spelling", correction);

25 }

26

27 abstract protected SpellingCorrector getSpellingCorrector();

28

29 }

1 package spellingCorrection;

2

3 import java.io.IOException;

5

6

7 public class AnotherImplementationOfSpellingCorrector {

8

9 private static final int INITIAL_LINKED_HASHSET_CAPACITY = 100000;

13 private List<String> wordsForCorrection = new LinkedList<>();

14

15 public AnotherImplementationOfSpellingCorrector() throws IOException {

16 SpellingUtils.initializeDictionaryListAndWordList(dictionarySet, wordsForCorrection);

110

111 public static void main(String[] args) throws IOException, InterruptedException {

112 AnotherImplementationOfSpellingCorrector spelling = new AnotherImplementationOfSpellingCorrector();

113

114

33 import org.apache.solr.common.SolrDocumentList;

34

35 //<start id="did-you-mean.corrector"/>

36 public class SpellCorrector {

41 private float threshold;

42

43 public SpellCorrector(StringDistance sd, float threshold)

44 throws MalformedURLException {

45 solr = new CommonsHttpSolrServer(

52 }

53

54 public String topSuggestion(String spelling)

55 throws SolrServerException {

56 query.setQuery("wordNGram:"+spelling); //<co id="co.dym.field"/>

63 SolrDocument doc = di.next();

64 String word = (String) doc.getFieldValue("word");

65 float distance = sd.getDistance(word, spelling); //<co id="co.dym.edit"/>

66 if (distance > maxDistance) {

67 maxDistance = distance;

1 package spelling.corrector;

2

3 import java.util.ArrayList;

6 import junit.framework.TestCase;

7

8 public abstract class SpellingCorrectorHelperTest extends

9 TestCase {

10

68 }

69

70 abstract protected SpellingCorrectorHelper getHelper();

71 }

72

1 package spelling.corrector;

2

3 import java.util.Collection;

4

5 public class StatSpellingCorrector implements SpellingCorrector {

6 final private SpellingCorrector corrector;

7 protected StatSpellingCorrector(SpellingCorrector corrector){

8 this.corrector = corrector;

10 protected StatSpellingCorrector(){

11 this(new DefaultSpellingCorrector(new StatSpellingCorrectorHelper()));

12 }

13 public String correct(String word) {

14 System.out.println("StatSpellingCorrector.correct()");

15 long begin = System.nanoTime();

16 String correction = this.corrector.correct(word);

9 import spelling.corrector.CorrectorBuilder;

10 import spelling.corrector.SpellingCorrector;

11 import spelling.dictionary.Dictionary;

15 public static void main(String[] args) throws IOException {

16 try {

17 SpellingCorrector sc = handleCommandLine(args);

18 Scanner inputScanner = new Scanner(System.in);

19 System.out.print("Please enter a word (anything else will quit the application):");

60 Dictionary.load(Locale.ENGLISH, new File(filename));

61

62 SpellingCorrector corrector = CorrectorBuilder.buildCorrector(isVerbose);

63 return corrector;

65

66 private static void showHelp() {

67 System.out.println("Usage SpellingCorrector [-h -f filename -v] \n\t-h : Show this help\n\t-f filename : Use this file to train the corrector\n\t-v [all] : Print statistic for methods if \"all\" then print statistic for every methods");

68 System.exit(0);

69 }

1 package spelling.corrector;

2

3 import java.util.ArrayList;

9 import spelling.dictionary.Dictionary;

10

11 public class DefaultSpellingCorrector implements SpellingCorrector {

12 private final Dictionary dictionnary;

13 private final SpellingCorrectorHelper helper;

14 protected DefaultSpellingCorrector(SpellingCorrectorHelper helper){

15 this(helper,Locale.ENGLISH);

16 }

17 protected DefaultSpellingCorrector(SpellingCorrectorHelper helper,Locale locale){

18 this.helper = helper;

19 this.dictionnary = Dictionary.getDictionnary(locale);

1 package spelling.corrector;

2

3 import java.util.Collection;

4 import java.util.List;

5

6 public interface SpellingCorrectorHelper {

7

8 /**

1 package spelling.corrector;

2

3 import java.util.Collection;

4 import java.util.List;

5

6 public class StatSpellingCorrectorHelper implements SpellingCorrectorHelper {

7 final private SpellingCorrectorHelper helper;

8

9 protected StatSpellingCorrectorHelper(SpellingCorrectorHelper helper) {

10 this.helper = helper;

11 }

12

13 protected StatSpellingCorrectorHelper() {

14 this(new DefaultSpellingCorrectorHelper());

1 package spelling.corrector;

2

3 import java.util.ArrayList;

8

9

10 public class DefaultSpellingCorrectorHelper implements SpellingCorrectorHelper {

11

12 public List<String> getDeletion(final String word) {

10 import cs276.util.Counter;

11

12 public class KGramSpellingCorrector implements SpellingCorrector {

13 /** Initializes spelling corrector by indexing kgrams in words from a file */

15 private HashMap<String, Counter<String>> index;

16

17 public KGramSpellingCorrector() {

18

19 // instantiate new index

10 import cs276.util.Counter;

11

12 public class KGramWithEditDistanceSpellingCorrector implements SpellingCorrector {

13 /** Initializes spelling corrector by indexing kgrams in words from a file */

14

15 KGramSpellingCorrector KGram;

16

17 public KGramWithEditDistanceSpellingCorrector() {

18 KGram = new KGramSpellingCorrector();

19 }

20

13 *

14 */

15 public class SpellingCorrector{

16 private static final Map<Integer, Set<String>> STATE_TOKENS = new HashMap<Integer, Set<String>>();

17 private static final Pattern DIGIT = Pattern.compile("^\\d+$");

29 * Attempts to correct possible state mis-spellings

30 * @param rawAddress

31 * @return rawAddress or spelling corrected address if a state mis-spelling is found

32 */

33 public static String correctStateSpelling(String rawAddress){

56 if(metrics == 1f){

57 return rawAddress;

58 }else if(metrics >= 0.75f){ //assume mis-spelling

59 if(i != 1){

60 for(int j=0; j<i-1; j++){

10 import cs276.util.Counter;

11

12 public class KGramWithEditAndFreqSpellingCorrector implements SpellingCorrector {

13 /** Initializes spelling corrector by indexing kgrams in words from a file */

14

15 KGramWithEditDistanceSpellingCorrector KGram;

16 KGramSpellingCorrector simpleKGram;

17

18 public KGramWithEditAndFreqSpellingCorrector() {

19 KGram = new KGramWithEditDistanceSpellingCorrector();

20 simpleKGram = new KGramSpellingCorrector();

21 }

22

10 import cs276.util.Counter;

11

12 public class KGramWithSoundexSpellingCorrector implements SpellingCorrector {

13 /** Initializes spelling corrector by indexing kgrams in words from a file */

14

15 KGramWithEditDistanceSpellingCorrector KGram;

16 KGramSpellingCorrector simpleKGram;

17

18 public KGramWithSoundexSpellingCorrector() {

19 KGram = new KGramWithEditDistanceSpellingCorrector();

20 simpleKGram = new KGramSpellingCorrector();

21 }

22

6 import java.util.regex.Pattern;

7

8 public class SpellingCorrector {

9 private HashMap<String, Integer> nWords = new HashMap<String, Integer>();

10 public SpellingCorrector(String path) throws IOException{

78 }

79 public static void main(String[] args)throws IOException {

80 SpellingCorrector sc = new SpellingCorrector("big.txt");

81 while(true){

82 System.out.println("Please input your word:");

19 import org.apache.lucene.store.FSDirectory;

20 import cs276.util.Counter;

21 import cs276.pe1.spell.KGramWithEditDistanceSpellingCorrector;

22 import cs276.util.StringUtils;

23

24 public class IMDBReader {

25

26 static KGramWithEditDistanceSpellingCorrector spellChecker = new KGramWithEditDistanceSpellingCorrector();

27

28 public static String runQueryForTitle(String rawQuery, String field) throws Exception {

4

5 /**

6 * Basic interface for the spelling corrector.

7 *

8 * @author dramage

9 */

10 public interface SpellingCorrector {

11

12 /**

13 * Returns list of possible spelling corrections for a given

14 * (possibly misspelled) word. The length of the returned list

15 * is up to the implementer, but enough high-scoring candidates

17 import org.rsna.ctp.pipeline.Processor;

18 import org.rsna.ctp.stdstages.anonymizer.AnonymizerStatus;

19 import org.rsna.ctp.stdstages.anonymizer.dicom.DICOMCorrector;

20 import org.rsna.server.User;

21 import org.rsna.util.FileUtil;

23

24 /**

25 * The DicomCorrector pipeline stage class.

26 */

27 public class DicomCorrector extends AbstractPipelineStage implements Processor, Scriptable {

28

29 static final Logger logger = Logger.getLogger(DicomCorrector.class);

30

31 File dicomScriptFile = null; //the DicomFilter script that determines whether to correct the object

21 import org.apache.lucene.search.spell.SpellChecker;

22

23 import cs276.pe1.spell.KGramSpellingCorrector;

24 import cs276.pe1.spell.KGramWithEditDistanceSpellingCorrector;

25

26 @SuppressWarnings("deprecation")

27 public class IMDBSpelling {

28 public static void main(String[] args) throws Exception {

29 //Read search query

34 //Spelling correct query

35 KGramWithEditDistanceSpellingCorrector k = new KGramWithEditDistanceSpellingCorrector();

36 List<String> suggest = k.corrections(search);

37 for(String s : suggest) search += " " + s;

12 * Scorer for evaluating a SpellingCorrector. Reads a file

13 * that contains correctly spelled words and, for each, one or

14 * more mis-spellings. A SpellingCorrector class is then

15 * instantiated (the specific type is passed as an argument

16 * and it is assumed to have a no-arg constructor).

28 System.err.println();

29 System.err.println("Usage: ");

30 System.err.println(" (java-cmd) SpellingScorer cs276.pe1.spell.YourSpellingCorrectorClass");

31 System.err.println();

32 System.err.println(message);

50

51 // instantiate scorer

52 SpellingCorrector scorer = null;

53 try {

54 Class<?> type = Class.forName(argv[0]);

13 import cs276.util.Counter;

14

15 public class KGramSpellingCorrector implements SpellingCorrector {

16 protected static int K = 2; //start with bigrams, then extend out

17 protected static int SE = K-1;//3;//defines extra kgrams - default to k-1 for best performance?

18 protected static int WL = 10;//returned Word List size

19 /** Initializes spelling corrector by indexing kgrams in words from a file */

20 protected Map<String,Set<String>> kgram;

21 protected Counter<String> freq;

22 public KGramSpellingCorrector() {

23 File path = new File("/afs/ir/class/cs276/pe1-2011/big.txt.gz");

24 //File path = new File("datasources/big.txt.gz");

14 import org.rsna.ctp.pipeline.Processor;

15 import org.rsna.ctp.stdstages.anonymizer.AnonymizerStatus;

16 import org.rsna.ctp.stdstages.anonymizer.dicom.DICOMCorrector;

17 import org.rsna.util.FileUtil;

18 import org.w3c.dom.Element;

21

22 /**

23 * The DicomCorrector pipeline stage class.

24 */

25 public class DicomCorrector extends AbstractPipelineStage implements Processor, Scriptable {

26

27 static final Logger logger = Logger.getLogger(DicomCorrector.class);

28

29 File dicomScriptFile = null; //the DicomFilter script that determines whether to anonymize the object

8 import cs276.util.Counters;

9

10 public class KGramWithEditDistanceSpellingCorrector extends KGramSpellingCorrector {

11 public enum ListCompare {

12 BASIC, //from the jaccard top 10, finds the one with the smallest edit distance (and highest jaccard score)

17 //Change this to change the type of corrections returned

18 protected static ListCompare correctionsType = ListCompare.NORMALIZE;

19 public KGramWithEditDistanceSpellingCorrector() {

20 super();

21 }

1 package spelling.corrector;

2

3 public class DefaultSpellingCorrectorTest extends SpellingCorrectorTest{

4

5 @Override

6 protected SpellingCorrector getSpellingCorrector() {

7 return new DefaultSpellingCorrector(new DefaultSpellingCorrectorHelper());

1 package spelling.corrector;

2

3

4 public class StatSpellingCorrectorHelperTest extends SpellingCorrectorHelperTest{

5

6 @Override

7 protected SpellingCorrectorHelper getHelper() {

8 return new StatSpellingCorrectorHelper();

1 package spelling.corrector;

2

3

4 public class StatSpellingCorrectorTest extends SpellingCorrectorTest {

5

6 @Override

7 protected SpellingCorrector getSpellingCorrector() {

8 return new StatSpellingCorrector();

1 package spelling.corrector;

2

3

4 public class DefaultSpellingCorrectorHelperTest extends SpellingCorrectorHelperTest {

5

6 @Override

7 protected SpellingCorrectorHelper getHelper() {

8 return new DefaultSpellingCorrectorHelper();

14 import java.util.List;

15

16 public interface ISpellingCorrector {

17 int getWordCount();

18

1 package spelling.corrector;

2

3 import java.util.Collection;

4

5 public interface SpellingCorrector {

6

7 /**

1 package spelling.corrector;

2

3

4 public class CorrectorBuilder {

5 private static SpellingCorrectorHelper buildHelper(boolean isVerbose){

6 SpellingCorrectorHelper helper = new DefaultSpellingCorrectorHelper();

7 if(isVerbose){

8 return new StatSpellingCorrectorHelper(helper);

14 public static SpellingCorrector buildCorrector(boolean isVerbose){

15 SpellingCorrector corrector = new DefaultSpellingCorrector(buildHelper(isVerbose));

16 if(isVerbose){

17 return new StatSpellingCorrector(corrector);