add studylist import option to attempt word deinflection

hlorenzi · Aug 18, 2024 · ad0830b · ad0830b
1 parent b6af56d
commit ad0830b
Show file tree

Hide file tree

Showing 9 changed files with 93 additions and 21 deletions.
diff --git a/backend/src/db/index.ts b/backend/src/db/index.ts
@@ -166,6 +166,7 @@ export interface Interface
     studylistWordImport: (
         authUser: Api.MaybeUser,
         studylistId: string,
+        attemptDeinflection: boolean,
         words: Api.StudylistWordImport.ImportWord[])
         => Promise<number[]>
 

diff --git a/backend/src/db/mongodb/index.ts b/backend/src/db/mongodb/index.ts
@@ -291,8 +291,8 @@ export async function connect(): Promise<Interface>
             MongoDbStudyLists.studylistWordAdd(state, authUser, studylistId, wordId),
         studylistWordRemoveMany: (authUser, studylistId, wordIds) =>
             MongoDbStudyLists.studylistWordRemoveMany(state, authUser, studylistId, wordIds),
-        studylistWordImport: (authUser, studylistId, words) =>
-            MongoDbStudyLists.studylistWordImport(state, authUser, studylistId, words),
+        studylistWordImport: (authUser, studylistId, attemptDeinflection, words) =>
+            MongoDbStudyLists.studylistWordImport(state, authUser, studylistId, attemptDeinflection, words),
         studylistWordsGet: (authUser, studylistId) =>
             MongoDbStudyLists.studylistWordsGet(state, authUser, studylistId),
         studylistCommunityGetRecent: (authUser, limit) =>

diff --git a/backend/src/db/mongodb/studylist.ts b/backend/src/db/mongodb/studylist.ts
@@ -1,4 +1,5 @@
 import * as MongoDriver from "mongodb"
+import * as MongoDb from "./index.ts"
 import * as Db from "../index.ts"
 import * as Auth from "../../auth/index.ts"
 import * as DbMongo from "./index.ts"
@@ -566,6 +567,7 @@ export async function studylistWordImport(
     state: DbMongo.State,
     authUser: Api.MaybeUser,
     studylistId: string,
+    attemptDeinflection: boolean,
     words: Api.StudylistWordImport.ImportWord[])
     : Promise<number[]>
 {
@@ -595,12 +597,20 @@ export async function studylistWordImport(
     const failedWordIndices: number[] = []
     for (let w = 0; w < words.length; w++)
     {
+        if (studylist.words.length + importedWords.length + 1 > Api.StudyList.maxWordCount)
+        {
+            failedWordIndices.push(w)
+            continue
+        }
+
         const word = words[w]
 
         const dbFind = word.reading ?
             { $all: [word.base, word.reading] } :
             word.base
 
+        let wordId: string | undefined = undefined
+
         const dbWords = await state.collWords
             .find({
                 [DbMongo.fieldWordLookUpHeadingsText]: dbFind,
@@ -610,15 +620,44 @@ export async function studylistWordImport(
             .limit(1)
             .toArray()
 
-        if (dbWords.length < 1 ||
-            studylist.words.length + importedWords.length + 1 > Api.StudyList.maxWordCount)
+        if (dbWords.length >= 1)
+            wordId = dbWords[0]._id
+
+        if (wordId === undefined &&
+            attemptDeinflection)
+        {
+            // Attempt to de-inflect word
+            const inflectionBreakdown = Inflection.breakdown(word.base)
+            const inflectionOf = Inflection.flattenBreakdown(inflectionBreakdown)
+
+            const dbFindQueries: any[] = []
+            for (const step of inflectionOf)
+            {
+                dbFindQueries.push({
+                    [MongoDb.fieldWordLookUpHeadingsText]: step.term,
+                    [MongoDb.fieldWordLookUpTags]: step.category,
+                })
+            }
+
+            const fieldLookUp = "lookUp" satisfies keyof MongoDb.DbWordEntry
+            const fieldLen = "len" satisfies keyof MongoDb.DbWordEntry["lookUp"]
+
+            const dbDeinflectedWord = await state.collWords
+                .find({ $or: dbFindQueries })
+                .sort({ [`${fieldLookUp}.${fieldLen}`]: -1, score: -1, _id: 1 })
+                .limit(1)
+                .toArray()
+
+            if (dbDeinflectedWord.length >= 1)
+                wordId = dbDeinflectedWord[0]._id
+        }
+
+        if (wordId === undefined)
         {
             failedWordIndices.push(w)
             continue
         }
 
-        const wordId = dbWords[0]._id
-
         if (studylist.words.find(w => w.id === wordId))
             continue
 

diff --git a/backend/src/server/search.ts b/backend/src/server/search.ts
@@ -392,20 +392,7 @@ function normalizeQuery(queryRaw: string): Api.Search.Query
         .filter(w => w.length !== 0)
 
     const inflectionBreakdown = Inflection.breakdown(queryJapanese)
-    const inflectionOf: Inflection.Inflected[] = []
-    const seenInflections = new Set<string>()
-    for (const step of inflectionBreakdown.flat())
-    {
-        const key = `${ step.sourceTerm };${ step.sourceCategory }`
-        if (seenInflections.has(key))
-            continue
-
-        seenInflections.add(key)
-        inflectionOf.push({
-            term: step.sourceTerm,
-            category: step.sourceCategory,
-        })
-    }
+    const inflectionOf = Inflection.flattenBreakdown(inflectionBreakdown)
 
     const queryCanBeDefinition =
         !Kana.hasJapanese(queryWithoutTags) ||

diff --git a/backend/src/server/studylist.ts b/backend/src/server/studylist.ts
@@ -206,6 +206,9 @@ export function init(
         if (typeof body.studylistId !== "string")
             throw Api.Error.malformed
 
+        if (typeof body.attemptDeinflection !== "boolean")
+            throw Api.Error.malformed
+
         if (!Array.isArray(body.words) ||
             !body.words.every(w =>
                 typeof w.base === "string" &&
@@ -215,6 +218,7 @@ export function init(
         const failedWordIndices = await db.studylistWordImport(
             await ServerAuth.authenticateRequest(auth, req),
             body.studylistId,
+            body.attemptDeinflection,
             body.words)
 
         res.send({ failedWordIndices } satisfies Api.StudylistWordImport.Response)

diff --git a/common/src/api/index.ts b/common/src/api/index.ts
@@ -495,6 +495,7 @@ export namespace StudylistWordImport
 
     export type Request = {
         studylistId: string
+        attemptDeinflection: boolean
         words: ImportWord[]
     }
 

diff --git a/common/src/inflection.ts b/common/src/inflection.ts
@@ -430,6 +430,29 @@ export function breakdown(
 }
 
 
+export function flattenBreakdown(
+    inflectionBreakdown: Breakdown)
+    : Inflected[]
+{
+    const inflectionOf: Inflected[] = []
+    const seenInflections = new Set<string>()
+    for (const step of inflectionBreakdown.flat())
+    {
+        const key = `${ step.sourceTerm };${ step.sourceCategory }`
+        if (seenInflections.has(key))
+            continue
+
+        seenInflections.add(key)
+        inflectionOf.push({
+            term: step.sourceTerm,
+            category: step.sourceCategory,
+        })
+    }
+
+    return inflectionOf
+}
+
+
 export function getRuleGroup(
     id: string)
     : Group | undefined

diff --git a/frontend/src/app.tsx b/frontend/src/app.tsx
@@ -24,6 +24,8 @@ export type Prefs = {
     studylistOrdering: "activity" | "name"
     studylistWordOrdering: "date-added" | "kana"
 
+    studylistImportAttemptDeinflection: boolean
+
     studylistExportHtmlCss: boolean
     studylistExportKanjiLevel: "common" | "jouyou" | "uncommon" | "rare" | "all"
     studylistExportSkipKatakana: boolean
@@ -47,6 +49,8 @@ export const prefsDefault: Prefs = {
     studylistOrdering: "activity",
     studylistWordOrdering: "date-added",
 
+    studylistImportAttemptDeinflection: false,
+
     studylistExportHtmlCss: true,
     studylistExportKanjiLevel: "common",
     studylistExportSkipKatakana: false,

diff --git a/frontend/src/pages/PageStudylist.tsx b/frontend/src/pages/PageStudylist.tsx
@@ -385,7 +385,8 @@ export function PageStudylist(props: Framework.RouteProps)
             }>
                 <HelpInfo>
                     Use the <Framework.IconBookmark color={ Framework.themeVar("iconGreenColor") }/> button
-                    displayed alongside the results of a search to add words to this study list!
+                    displayed alongside the results of a search to add words to this study list!<br/>
+                    Alternatively, use the Import button above to add words listed in a file.
                 </HelpInfo>
             </Solid.Show>
 
@@ -795,6 +796,7 @@ function ImportPopup(props: {
                 {
                     const res = await App.Api.studylistWordImport({
                         studylistId: props.studylist.id,
+                        attemptDeinflection: App.usePrefs().studylistImportAttemptDeinflection,
                         words: words.slice(i, i + packLen),
                     })
 
@@ -820,6 +822,7 @@ function ImportPopup(props: {
                             "\n"
                     }
 
+                    console.error(text)
                     alert(text)
                 }
                 else
@@ -894,6 +897,16 @@ function ImportPopup(props: {
             </ImportWordGrid>
 
             <br/>
+
+            <Framework.Checkbox
+                label="Attempt de-inflection"
+                value={ () => App.usePrefs().studylistImportAttemptDeinflection }
+                onChange={ (value) => App.mergePrefs({ studylistImportAttemptDeinflection: value }) }
+            />
+
+            <br/>
+            <br/>
+
             <Framework.Button
                 icon={ <Framework.IconUpload/> }
                 label="Import"