Merge pull request #41 from domino14/CSW24

add CSW24
domino14 · Nov 25, 2024 · 814d8e1 · 814d8e1
2 parents 438beae + f1e0ad1
commit 814d8e1
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 6 deletions.
diff --git a/cmd/cwl/main.go b/cmd/cwl/main.go
@@ -18,7 +18,7 @@ import (
 )
 
 const (
-	BritishDict  = "CSW21"
+	BritishDict  = "CSW24"
 	AmericanDict = "NWL23"
 )
 

diff --git a/cmd/searchserver/text_handler.go b/cmd/searchserver/text_handler.go
@@ -84,7 +84,7 @@ func anagram(anagramserver *anagramserver.Server, w http.ResponseWriter, r *http
 	}
 	lexicon, ok := r.URL.Query()["lexicon"]
 	if !ok || len(lexicon[0]) < 1 {
-		lexicon = []string{"CSW21"}
+		lexicon = []string{"CSW24"}
 	}
 	res, err := anagramserver.Anagram(r.Context(), connect.NewRequest(&wordsearcher.AnagramRequest{
 		Lexicon: lexicon[0],
@@ -105,7 +105,7 @@ func define(wsServer *searchserver.WordSearchServer, w http.ResponseWriter, r *h
 	}
 	lexicon, ok := r.URL.Query()["lexicon"]
 	if !ok || len(lexicon[0]) < 1 {
-		lexicon = []string{"CSW21"}
+		lexicon = []string{"CSW24"}
 	}
 	res, err := wsServer.GetWordInformation(r.Context(), connect.NewRequest(&wordsearcher.DefineRequest{
 		Lexicon: lexicon[0], Word: word[0],
@@ -129,7 +129,7 @@ func patternSearch(wsServer *searchserver.WordSearchServer, w http.ResponseWrite
 	}
 	lexicon, ok := r.URL.Query()["lexicon"]
 	if !ok || len(lexicon[0]) < 1 {
-		lexicon = []string{"CSW21"}
+		lexicon = []string{"CSW24"}
 	}
 	res, err := wsServer.WordSearch(r.Context(), connect.NewRequest(&wordsearcher.WordSearchRequest{
 		Lexicon: lexicon[0], Glob: pattern[0], AppliesTo: "word",
@@ -153,7 +153,7 @@ func definitionSearch(wsServer *searchserver.WordSearchServer, w http.ResponseWr
 	}
 	lexicon, ok := r.URL.Query()["lexicon"]
 	if !ok || len(lexicon[0]) < 1 {
-		lexicon = []string{"CSW21"}
+		lexicon = []string{"CSW24"}
 	}
 	res, err := wsServer.WordSearch(r.Context(), connect.NewRequest(&wordsearcher.WordSearchRequest{
 		Lexicon: lexicon[0], Glob: "*" + pattern[0] + "*", AppliesTo: "definition",

diff --git a/dbmaker/dbmaker.go b/dbmaker/dbmaker.go
@@ -756,8 +756,17 @@ func populateAlphsDefs(filename string, combinations func(string, bool) uint64,
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		fields := strings.Fields(scanner.Text())
+
 		if len(fields) > 0 {
-			word := common.InitializeWord(strings.ToUpper(fields[0]), dist)
+			rawWord := fields[0]
+			rawWord = strings.ReplaceAll(rawWord, "1", "[CH]")
+			rawWord = strings.ReplaceAll(rawWord, "2", "[LL]")
+			rawWord = strings.ReplaceAll(rawWord, "3", "[RR]")
+
+			// Spanish hack. Older versions of FISE files were given to us with
+			// 1, 2, 3 instead of [CH], [LL], [RR].
+
+			word := common.InitializeWord(strings.ToUpper(rawWord), dist)
 			definition := ""
 			if len(fields) > 1 {
 				definition = strings.Join(fields[1:], " ")

diff --git a/dbmaker/utils.go b/dbmaker/utils.go
@@ -83,6 +83,15 @@ func LexiconMappings(dataPath string) LexiconMap {
 			LetterDistribution: englishLD,
 			Difficulties:       createDifficultyMap(lexiconPath, "CSW21"),
 		},
+		{
+			LexiconName:        "CSW24",
+			LexiconFilename:    filepath.Join(lexiconPath, "CSW24.txt"),
+			KWG:                loadKWG(dataPath, "CSW24"),
+			LexiconIndex:       25,
+			DescriptiveName:    "Collins 2024",
+			LetterDistribution: englishLD,
+			Difficulties:       createDifficultyMap(lexiconPath, "CSW24"),
+		},
 	}
 
 	fiseFamily := []*LexiconInfo{
-Original file line number
+Diff line change
@@ Expand Up / @@ -18,7 +18,7 @@ import ( @@
     )
     const (
-    	BritishDict  = "CSW21"
+    	BritishDict  = "CSW24"
     	AmericanDict = "NWL23"
     )
@@ Expand Down @@