グライド入力の実装の試み

app/build.gradle

@@ -31,8 +31,8 @@ android {
         applicationId "com.kazumaproject.markdownhelperkeyboard"
         minSdk 24
         targetSdk 36
-        versionCode 747
-        versionName "1.7.53"
+        versionCode 749
+        versionName "1.7.55"
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
 

app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/english/LoadEnglishLOUDSTest.kt

@@ -1,10 +1,13 @@
 package com.kazumaproject.markdownhelperkeyboard.english
 
 import androidx.test.platform.app.InstrumentationRegistry
+import com.kazumaproject.markdownhelperkeyboard.converter.bitset.SuccinctBitVector
+import com.kazumaproject.markdownhelperkeyboard.converter.english.louds.louds_with_term_id.LOUDSWithTermId
 import org.junit.Before
 import org.junit.Test
 import java.io.BufferedInputStream
 import java.io.ObjectInputStream
+import java.util.zip.ZipInputStream
 
 class LoadEnglishLOUDSTest {
     @Before
@@ -16,22 +19,24 @@ class LoadEnglishLOUDSTest {
     fun testLoadEnglishLOUDS() {
         val context = InstrumentationRegistry.getInstrumentation().targetContext
 
-        val objectInput = ObjectInputStream(
-            BufferedInputStream(context.assets.open("english/english.dat"))
-        )
-        val result = EnglishLOUDS().readExternal(objectInput)
-        println("Loaded object: ${result.costListSave.size}")
-        objectInput.close()
+        val zipInputStream = ZipInputStream(context.assets.open("english/reading.dat.zip"))
+        zipInputStream.nextEntry
+        val result = ObjectInputStream(BufferedInputStream(zipInputStream)).use { objectInput ->
+            LOUDSWithTermId().readExternalNotCompress(objectInput)
+        }
+        val succinctBitVector = SuccinctBitVector(result.LBS)
+        val leafBitVector = SuccinctBitVector(result.isLeaf)
+        println("Loaded object: ${result.labels.size}")
 
         val text = "on"
-        val commonPrefixSearch = result.commonPrefixSearch(text)
+        val commonPrefixSearch = result.commonPrefixSearch(text, succinctBitVector)
         val searchResult = commonPrefixSearch.map {
-            Pair(it, result.getTermId(result.getNodeIndex(it)))
+            Pair(it, result.getTermId(result.getNodeIndex(it, succinctBitVector), leafBitVector))
         }
         println(searchResult)
-        val suggestions = result.predictiveSearch("i", 4)
+        val suggestions = result.predictiveSearch("i", succinctBitVector, 4)
         val pairs = suggestions.map { term ->
-            term to result.getTermId(result.getNodeIndex(term))
+            term to result.getTermId(result.getNodeIndex(term, succinctBitVector), leafBitVector)
         }.toMutableList()
         pairs.sortBy { it.second }
         println(pairs)

app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/engine/EnglishEngine.kt

@@ -5,16 +5,42 @@ import com.kazumaproject.markdownhelperkeyboard.converter.candidate.Candidate
 import com.kazumaproject.markdownhelperkeyboard.converter.english.louds.LOUDS
 import com.kazumaproject.markdownhelperkeyboard.converter.english.louds.louds_with_term_id.LOUDSWithTermId
 import com.kazumaproject.markdownhelperkeyboard.converter.english.tokenArray.TokenArray
+import com.kazumaproject.markdownhelperkeyboard.BuildConfig
+import com.kazumaproject.markdownhelperkeyboard.converter.glide.QwertyGlideDecodeOptions
+import com.kazumaproject.markdownhelperkeyboard.converter.glide.QwertyGlideDecoder
+import com.kazumaproject.markdownhelperkeyboard.converter.glide.QwertyGlideDecodeMetrics
+import com.kazumaproject.markdownhelperkeyboard.converter.glide.QwertyGlideDictionaryEntry
+import com.kazumaproject.markdownhelperkeyboard.converter.glide.QwertyGlideCandidateCaseExpander
+import com.kazumaproject.markdownhelperkeyboard.converter.glide.QwertyGlideCandidateProvider
+import com.kazumaproject.markdownhelperkeyboard.converter.glide.QwertyGlideIndexedDictionaryProvider
+import com.kazumaproject.qwerty_keyboard.glide.QwertyInputPointers
+import com.kazumaproject.qwerty_keyboard.glide.QwertyKeyboardProximityInfo
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
 import timber.log.Timber
 
-class EnglishEngine {
+class EnglishEngine : QwertyGlideCandidateProvider {
     private lateinit var readingLOUDS: LOUDSWithTermId
     private lateinit var wordLOUDS: LOUDS
     private lateinit var tokenArray: TokenArray
     private lateinit var succinctBitVectorLBSReading: SuccinctBitVector
     private lateinit var succinctBitVectorReadingIsLeaf: SuccinctBitVector
     private lateinit var succinctBitVectorTokenArray: SuccinctBitVector
     private lateinit var succinctBitVectorLBSWord: SuccinctBitVector
+    @Volatile
+    private var qwertyGlideDecoder: QwertyGlideDecoder? = null
+    @Volatile
+    private var qwertyFallbackGlideDecoder: QwertyGlideDecoder? = null
+    @Volatile
+    private var qwertyGlideDictionaryReady: Boolean = false
+    @Volatile
+    private var qwertyGlideWarmupJob: Job? = null
+    private val qwertyGlideCandidateCaseExpander = QwertyGlideCandidateCaseExpander()
+    private val qwertyGlideWarmupScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
 
     companion object {
         const val LENGTH_MULTIPLY = 2000
@@ -36,6 +62,162 @@ class EnglishEngine {
         this.succinctBitVectorLBSWord = englishSuccinctBitVectorLBSWord
         this.succinctBitVectorReadingIsLeaf = englishSuccinctBitVectorReadingIsLeaf
         this.succinctBitVectorTokenArray = englishSuccinctBitVectorTokenArray
+        qwertyGlideDictionaryReady = false
+        qwertyGlideDecoder = null
+        qwertyFallbackGlideDecoder = createQwertyGlideDecoder(
+            entries = fallbackGlideDictionaryEntries(),
+            dictionaryReady = false
+        )
+    }
+
+    override suspend fun getGlideCandidates(
+        inputPointers: QwertyInputPointers,
+        proximityInfo: QwertyKeyboardProximityInfo,
+        previousText: String,
+        limit: Int
+    ): List<Candidate> {
+        if (limit <= 0 || inputPointers.points.size < 2 || proximityInfo.keys.isEmpty()) return emptyList()
+        val decoder = qwertyGlideDecoder ?: run {
+            warmUpQwertyGlideDecoderAsync()
+            getOrCreateFallbackQwertyGlideDecoder()
+        }
+        val baseCandidates = decoder.decode(
+            inputPointers = inputPointers,
+            proximityInfo = proximityInfo,
+            previousText = previousText,
+            limit = (limit * 3).coerceAtLeast(limit)
+        )
+        return qwertyGlideCandidateCaseExpander.expand(baseCandidates, limit)
+    }
+
+    fun warmUpQwertyGlideDecoderAsync() {
+        if (qwertyGlideDictionaryReady && qwertyGlideDecoder != null) return
+        synchronized(this) {
+            val existing = qwertyGlideWarmupJob
+            if (existing?.isActive == true) return
+            qwertyGlideWarmupJob = qwertyGlideWarmupScope.launch {
+                val startedAt = System.nanoTime()
+                val entries = buildGlideDictionaryEntries()
+                val decoder = createQwertyGlideDecoder(
+                    entries = entries,
+                    dictionaryReady = true
+                )
+                qwertyGlideDecoder = decoder
+                qwertyGlideDictionaryReady = true
+                if (BuildConfig.DEBUG) {
+                    Timber.d(
+                        "QWERTY glide dictionary warmup complete: entries=${entries.size} elapsed_ms=${(System.nanoTime() - startedAt) / 1_000_000L}"
+                    )
+                }
+            }
+        }
+    }
+
+    fun isQwertyGlideDictionaryReady(): Boolean = qwertyGlideDictionaryReady
+
+    fun cancelQwertyGlideWarmup() {
+        qwertyGlideWarmupJob?.cancel()
+        qwertyGlideWarmupJob = null
+    }
+
+    fun releaseQwertyGlideResources() {
+        cancelQwertyGlideWarmup()
+        qwertyGlideDecoder = null
+        qwertyFallbackGlideDecoder = null
+        qwertyGlideDictionaryReady = false
+    }
+
+    fun invalidateQwertyGlideCache() {
+        qwertyGlideDecoder?.clearCache()
+        qwertyFallbackGlideDecoder?.clearCache()
+    }
+
+    private fun getOrCreateFallbackQwertyGlideDecoder(): QwertyGlideDecoder {
+        qwertyFallbackGlideDecoder?.let { return it }
+        return synchronized(this) {
+            qwertyFallbackGlideDecoder ?: createQwertyGlideDecoder(
+                entries = fallbackGlideDictionaryEntries(),
+                dictionaryReady = false
+            ).also { qwertyFallbackGlideDecoder = it }
+        }
+    }
+
+    private fun createQwertyGlideDecoder(
+        entries: Iterable<QwertyGlideDictionaryEntry>,
+        dictionaryReady: Boolean
+    ): QwertyGlideDecoder {
+        return QwertyGlideDecoder(
+            dictionaryProvider = QwertyGlideIndexedDictionaryProvider(entries),
+            options = QwertyGlideDecodeOptions(),
+            dictionaryReady = dictionaryReady,
+            metricsListener = ::logQwertyGlideMetrics
+        )
+    }
+
+    private fun logQwertyGlideMetrics(metrics: QwertyGlideDecodeMetrics) {
+        if (!BuildConfig.DEBUG) return
+        Timber.d(
+            "QWERTY glide decode: dictionary_ready=${metrics.dictionaryReady} " +
+                    "raw_bucket_candidate_count=${metrics.rawBucketCandidateCount} " +
+                    "prefilter_candidate_count=${metrics.prefilterCandidateCount} " +
+                    "full_score_candidate_count=${metrics.fullScoreCandidateCount} " +
+                    "rerank_candidate_count=${metrics.rerankCandidateCount} " +
+                    "decode_total_ms=${metrics.decodeTotalMs} prefilter_ms=${metrics.prefilterMs} " +
+                    "full_score_ms=${metrics.fullScoreMs} rerank_ms=${metrics.rerankMs} " +
+                    "cache_hit=${metrics.cacheHit}"
+        )
+    }
+
+    private fun fallbackGlideDictionaryEntries(): List<QwertyGlideDictionaryEntry> {
+        return listOf(
+            "hello", "good", "test", "word", "world", "keyboard", "android", "sumire",
+            "coffee", "letter", "people", "glide", "time", "home", "something"
+        ).map { word -> QwertyGlideDictionaryEntry(word, 6000) }
+    }
+
+    private fun buildGlideDictionaryEntries(): List<QwertyGlideDictionaryEntry> {
+        val entries = linkedMapOf<String, QwertyGlideDictionaryEntry>()
+        val readings = readingLOUDS.predictiveSearch(
+            prefix = "",
+            succinctBitVector = succinctBitVectorLBSReading
+        )
+        for (reading in readings) {
+            if (reading.length !in 2..24 || !reading.all { it in 'a'..'z' }) continue
+            val nodeIndex = readingLOUDS.getNodeIndex(
+                reading,
+                succinctBitVector = succinctBitVectorLBSReading
+            )
+            if (nodeIndex <= 0) continue
+            val termId = readingLOUDS.getTermId(
+                nodeIndex,
+                succinctBitVector = succinctBitVectorReadingIsLeaf
+            )
+            if (termId < 0) continue
+            val tokens = tokenArray.getListDictionaryByYomiTermId(
+                termId,
+                succinctBitVector = succinctBitVectorTokenArray
+            )
+            if (tokens.isEmpty()) {
+                entries.mergeGlideEntry(reading, 9000)
+            } else {
+                for (entry in tokens) {
+                    val word = when (entry.nodeId) {
+                        -1 -> reading
+                        else -> wordLOUDS.getLetter(
+                            entry.nodeId,
+                            succinctBitVector = succinctBitVectorLBSWord
+                        )
+                    }.lowercase()
+                    if (word.length in 2..24 && word.all { it in 'a'..'z' }) {
+                        entries.mergeGlideEntry(word, entry.wordCost.toInt())
+                    }
+                }
+            }
+        }
+        fallbackGlideDictionaryEntries().forEach { entry ->
+            entries.mergeGlideEntry(entry.word, entry.wordCost)
+        }
+        return entries.values.toList()
     }
 
     fun getCandidates(
@@ -292,3 +474,14 @@ class EnglishEngine {
     }
 
 }
+
+private fun MutableMap<String, QwertyGlideDictionaryEntry>.mergeGlideEntry(
+    word: String,
+    wordCost: Int
+) {
+    val normalizedWord = word.lowercase()
+    val current = this[normalizedWord]
+    if (current == null || wordCost < current.wordCost) {
+        this[normalizedWord] = QwertyGlideDictionaryEntry(normalizedWord, wordCost)
+    }
+}

app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/glide/QwertyGlideCandidateCaseExpander.kt

@@ -0,0 +1,55 @@
+package com.kazumaproject.markdownhelperkeyboard.converter.glide
+
+import com.kazumaproject.markdownhelperkeyboard.converter.candidate.Candidate
+import java.util.Locale
+
+class QwertyGlideCandidateCaseExpander {
+    fun expand(
+        candidates: List<Candidate>,
+        limit: Int
+    ): List<Candidate> {
+        if (candidates.isEmpty() || limit <= 0) return emptyList()
+
+        val lowestScoreByString = LinkedHashMap<String, Candidate>()
+        fun add(candidate: Candidate) {
+            val existing = lowestScoreByString[candidate.string]
+            if (existing == null || candidate.score < existing.score) {
+                lowestScoreByString[candidate.string] = candidate
+            }
+        }
+
+        for (candidate in candidates) {
+            add(candidate)
+            if (candidate.string.isEmpty()) continue
+
+            val capitalized = candidate.string.replaceFirstChar { char ->
+                char.uppercase(Locale.ROOT)
+            }
+            add(
+                candidate.copy(
+                    string = capitalized,
+                    length = capitalized.length.toUByte(),
+                    score = candidate.score + CAPITALIZED_SCORE_OFFSET
+                )
+            )
+
+            val uppercase = candidate.string.uppercase(Locale.ROOT)
+            add(
+                candidate.copy(
+                    string = uppercase,
+                    length = uppercase.length.toUByte(),
+                    score = candidate.score + UPPERCASE_SCORE_OFFSET
+                )
+            )
+        }
+
+        return lowestScoreByString.values
+            .sortedBy { it.score }
+            .take(limit)
+    }
+
+    companion object {
+        const val CAPITALIZED_SCORE_OFFSET = 1500
+        const val UPPERCASE_SCORE_OFFSET = 3000
+    }
+}