Seed canonical morph types and regenerate search index

backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/SortingTests.cs

@@ -1,4 +1,7 @@
+using FwDataMiniLcmBridge.Api;
+using FwDataMiniLcmBridge.LcmUtils;
 using FwDataMiniLcmBridge.Tests.Fixtures;
+using MiniLcm.Models;
 
 namespace FwDataMiniLcmBridge.Tests.MiniLcmTests;
 
@@ -9,4 +12,42 @@ protected override Task<IMiniLcmApi> NewApi()
     {
         return Task.FromResult<IMiniLcmApi>(fixture.NewProjectApi("sorting-test", "en", "en"));
     }
+
+    [Theory]
+    [InlineData("aaaa", SortField.Headword)] // FTS
+    [InlineData("a", SortField.Headword)] // non-FTS
+    [InlineData("aaaa", SortField.SearchRelevance)] // FTS
+    [InlineData("a", SortField.SearchRelevance)] // non-FTS
+    public async Task SecondaryOrder_DefaultsToStem(string query, SortField sortField)
+    {
+        var otherMorphTypeEntryId = Guid.NewGuid();
+        Entry[] expected = [
+            new() { Id = otherMorphTypeEntryId, LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Unknown }, // SecondaryOrder defaults to Stem = 1
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.BoundStem }, // SecondaryOrder = 2
+            new() { Id = Guid.NewGuid(), LexemeForm = { ["en"] = "aaaa" }, MorphType = MorphTypeKind.Suffix }, // SecondaryOrder = 6
+        ];
+
+        var ids = expected.Select(e => e.Id).ToHashSet();
+
+        foreach (var entry in Faker.Faker.Random.Shuffle(expected))
+            await Api.CreateEntry(entry);
+
+        var fwDataApi = (BaseApi as FwDataMiniLcmApi)!;
+        await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Clear morph type",
+            "Revert morph type",
+            () =>
+            {
+                // the fwdata api doesn't allow creating entries with MorphType.Other or Unknown, so we force it
+                var otherMorphTypeEntry = fwDataApi.EntriesRepository.GetObject(otherMorphTypeEntryId);
+                otherMorphTypeEntry.LexemeFormOA.MorphTypeRA = null;
+                return ValueTask.CompletedTask;
+            });
+
+        var results = (await Api.SearchEntries(query, new(new(sortField))).ToArrayAsync())
+            .Where(e => ids.Contains(e.Id))
+            .ToList();
+
+        results.Should().BeEquivalentTo(expected,
+            options => options.WithStrictOrdering());
+    }
 }

backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs

@@ -652,7 +652,7 @@ private Entry FromLexEntry(ILexEntry entry)
     {
         try
         {
-            return new Entry
+            var result = new Entry
             {
                 Id = entry.Guid,
                 Note = FromLcmMultiString(entry.Comment),
@@ -670,6 +670,7 @@ private Entry FromLexEntry(ILexEntry entry)
                 // ILexEntry.PublishIn is a virtual property that inverts DoNotPublishInRC against all publications
                 PublishIn = entry.PublishIn.Select(FromLcmPossibility).ToList(),
             };
+            return result;
         }
         catch (Exception e)
         {
@@ -939,12 +940,13 @@ private IEnumerable<ILexEntry> GetFilteredAndSortedEntries(Func<ILexEntry, bool>
     private IEnumerable<ILexEntry> ApplySorting(SortOptions order, IEnumerable<ILexEntry> entries, string? query)
     {
         var sortWs = GetWritingSystemHandle(order.WritingSystem, WritingSystemType.Vernacular);
+        var stemSecondaryOrder = MorphTypeRepository.GetObject(MoMorphTypeTags.kguidMorphStem).SecondaryOrder;
         if (order.Field == SortField.SearchRelevance)
         {
-            return entries.ApplyRoughBestMatchOrder(order, sortWs, query);
+            return entries.ApplyRoughBestMatchOrder(order, sortWs, stemSecondaryOrder, query);
         }
 
-        return order.ApplyOrder(entries, e => e.LexEntryHeadword(sortWs));
+        return entries.ApplyHeadwordOrder(order, sortWs, stemSecondaryOrder);
     }
 
     public IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options = null)
@@ -956,7 +958,7 @@ public IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options
     private Func<ILexEntry, bool>? EntrySearchPredicate(string? query = null)
     {
         if (string.IsNullOrEmpty(query)) return null;
-        return entry => entry.CitationForm.SearchValue(query) ||
+        return entry => entry.SearchHeadWord(query) || // CitationForm.SearchValue would be redundant
                         entry.LexemeFormOA?.Form.SearchValue(query) is true ||
                         entry.AllSenses.Any(s => s.Gloss.SearchValue(query));
     }

backend/FwLite/FwDataMiniLcmBridge/Api/LcmHelpers.cs

@@ -11,7 +11,7 @@ namespace FwDataMiniLcmBridge.Api;
 
 internal static class LcmHelpers
 {
-    internal static string? LexEntryHeadword(this ILexEntry entry, int? ws = null)
+    internal static string? LexEntryHeadword(this ILexEntry entry, int? ws = null, bool applyMorphTokens = true)
     {
         var citationFormTs =
             ws.HasValue ? entry.CitationForm.get_String(ws.Value)
@@ -27,7 +27,12 @@ internal static class LcmHelpers
             : null;
         var lexemeForm = lexemeFormTs?.Text?.Trim(WhitespaceChars);
 
-        return lexemeForm;
+        if (string.IsNullOrEmpty(lexemeForm) || !applyMorphTokens) return lexemeForm;
+
+        var morphType = entry.LexemeFormOA?.MorphTypeRA;
+        var leading = morphType?.Prefix ?? "";
+        var trailing = morphType?.Postfix ?? "";
+        return (leading + lexemeForm + trailing).Trim(WhitespaceChars);
     }
 
     internal static string LexEntryHeadwordOrUnknown(this ILexEntry entry, int? ws = null)
@@ -36,6 +41,19 @@ internal static string LexEntryHeadwordOrUnknown(this ILexEntry entry, int? ws =
         return string.IsNullOrEmpty(headword) ? Entry.UnknownHeadword : headword;
     }
 
+    internal static bool SearchHeadWord(this ILexEntry entry, string value)
+    {
+        foreach (var ws in entry.Cache.ServiceLocator.WritingSystems.VernacularWritingSystems)
+        {
+            var headword = entry.HeadWordForWs(ws.Handle);
+            if (headword is null) continue;
+            var text = headword.Text;
+            if (string.IsNullOrEmpty(text)) continue;
+            if (text.ContainsDiacriticMatch(value)) return true;
+        }
+        return false;
+    }
+
     internal static bool SearchValue(this ITsMultiString multiString, string value)
     {
         for (var i = 0; i < multiString.StringCount; i++)

backend/FwLite/FwDataMiniLcmBridge/Api/Sorting.cs

@@ -6,31 +6,55 @@ namespace FwDataMiniLcmBridge.Api;
 
 internal static class Sorting
 {
+    public static IEnumerable<ILexEntry> ApplyHeadwordOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, int stemSecondaryOrder)
+    {
+        if (order.Ascending)
+        {
+            return entries
+                .OrderBy(e => e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false))
+                .ThenBy(e => e.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
+                .ThenBy(e => e.HomographNumber)
+                .ThenBy(e => e.Id.Guid);
+        }
+        else
+        {
+            return entries
+                .OrderByDescending(e => e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false))
+                .ThenByDescending(e => e.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
+                .ThenByDescending(e => e.HomographNumber)
+                .ThenByDescending(e => e.Id.Guid);
+        }
+    }
+
     /// <summary>
     /// Rough emulation of FTS search relevance. Headword matches come first, preferring
     /// prefix matches (e.g. when searching "tan" then "tanan" is before "matan"), then shorter, then alphabetical.
     /// See also: EntrySearchService.FilterAndRank for the FTS-based equivalent in LcmCrdt.
     /// </summary>
-    public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, string? query = null)
+    public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, int stemSecondaryOrder, string? query = null)
     {
-        var projected = entries.Select(e => (Entry: e, Headword: e.LexEntryHeadword(sortWsHandle)));
+        var projected = entries.Select(e => (Entry: e, Headword: e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false)));
         if (order.Ascending)
         {
             return projected
-                .OrderByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
-                .ThenByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .OrderByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .ThenByDescending(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
                 .ThenBy(x => x.Headword?.Length ?? 0)
                 .ThenBy(x => x.Headword)
+                .ThenBy(x => x.Entry.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
+                .ThenBy(x => x.Entry.HomographNumber)
                 .ThenBy(x => x.Entry.Id.Guid)
                 .Select(x => x.Entry);
         }
         else
         {
             return projected
-                .OrderBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
-                .ThenBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .OrderBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.StartsWithDiacriticMatch(query!) ?? false))
+                .ThenBy(x => !string.IsNullOrEmpty(query) && (x.Headword?.ContainsDiacriticMatch(query!) ?? false))
                 .ThenByDescending(x => x.Headword?.Length ?? 0)
                 .ThenByDescending(x => x.Headword)
+                .ThenByDescending(x => x.Entry.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
+                .ThenByDescending(x => x.Entry.HomographNumber)
                 .ThenByDescending(x => x.Entry.Id.Guid)
                 .Select(x => x.Entry);
         }

backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateMorphTypeProxy.cs

@@ -1,3 +1,4 @@
+using System.Diagnostics.CodeAnalysis;
 using MiniLcm.Models;
 using SIL.LCModel;
 
@@ -8,11 +9,13 @@ public class UpdateMorphTypeProxy : MorphType
     private readonly IMoMorphType _lcmMorphType;
     private readonly FwDataMiniLcmApi _lexboxLcmApi;
 
+    [SetsRequiredMembers]
     public UpdateMorphTypeProxy(IMoMorphType lcmMorphType, FwDataMiniLcmApi lexboxLcmApi)
     {
         _lcmMorphType = lcmMorphType;
         Id = lcmMorphType.Guid;
         _lexboxLcmApi = lexboxLcmApi;
+        Kind = LcmHelpers.FromLcmMorphType(lcmMorphType);
     }
 
     public override MultiString Name

backend/FwLite/FwLiteProjectSync.Tests/FluentAssertGlobalConfig.cs

@@ -1,3 +1,4 @@
+using System.Runtime.CompilerServices;
 using FluentAssertions.Extensibility;
 using FwLiteProjectSync.Tests;
 
@@ -7,6 +8,12 @@ namespace FwLiteProjectSync.Tests;
 
 public static class FluentAssertGlobalConfig
 {
+    [ModuleInitializer]
+    internal static void InitVerify()
+    {
+        VerifierSettings.OmitContentFromException();
+    }
+
     public static void Initialize()
     {
         MiniLcm.Tests.FluentAssertGlobalConfig.Initialize();

backend/FwLite/FwLiteProjectSync.Tests/ProjectSnapshotSerializationTests.cs

@@ -54,6 +54,7 @@ public async Task AssertSena3Snapshots(string sourceSnapshotName)
     }
 
     [Fact]
+    [Trait("Category", "Verified")]
     public async Task LatestSena3SnapshotRoundTrips()
     {
         // arrange

backend/FwLite/FwLiteProjectSync.Tests/Sena3SyncTests.cs

@@ -96,6 +96,37 @@ private async Task WorkaroundMissingWritingSystems()
 
     }
 
+    [Fact]
+    [Trait("Category", "Integration")]
+    public async Task CanonicalMorphTypes_MatchFwDataMorphTypes()
+    {
+        var fwDataMorphTypes = await _fwDataApi.GetMorphTypes().ToArrayAsync();
+        fwDataMorphTypes.Should().NotBeEmpty("Sena 3 should have morph types");
+
+        // Verify every FwData morph type has a matching canonical entry
+        foreach (var fwMorphType in fwDataMorphTypes)
+        {
+            if (fwMorphType.Kind == MorphTypeKind.Unknown)
+                continue;
+
+            CanonicalMorphTypes.All.Should().ContainKey(fwMorphType.Kind,
+                $"canonical morph types should include {fwMorphType.Kind}");
+            var canonical = CanonicalMorphTypes.All[fwMorphType.Kind];
+            canonical.Id.Should().Be(fwMorphType.Id, $"GUID for {fwMorphType.Kind} should match FwData");
+            canonical.Prefix.Should().Be(fwMorphType.Prefix, $"Prefix for {fwMorphType.Kind} should match FwData");
+            canonical.Postfix.Should().Be(fwMorphType.Postfix, $"Postfix for {fwMorphType.Kind} should match FwData");
+            canonical.SecondaryOrder.Should().Be(fwMorphType.SecondaryOrder, $"SecondaryOrder for {fwMorphType.Kind} should match FwData");
+        }
+
+        // Verify every canonical morph type exists in FwData (no extras we shouldn't have)
+        var fwDataKinds = fwDataMorphTypes
+            .Where(m => m.Kind != MorphTypeKind.Unknown)
+            .Select(m => m.Kind)
+            .ToHashSet();
+        CanonicalMorphTypes.All.Keys.Should().BeSubsetOf(fwDataKinds,
+            "every canonical morph type should exist in the Sena 3 FwData project");
+    }
+
     [Fact]
     [Trait("Category", "Integration")]
     public async Task DryRunImport_MakesNoChanges()
@@ -207,6 +238,7 @@ public async Task SecondSena3SyncDoesNothing()
     /// </summary>
     [Fact]
     [Trait("Category", "Integration")]
+    [Trait("Category", "Verified")]
     public async Task LiveSena3Sync()
     {
         // arrange - put "live" crdt db and fw-headless snapshot in place

backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs

@@ -702,4 +702,32 @@ public async Task CanCreateAComplexFormTypeAndSyncsIt()
 
         _fixture.FwDataApi.GetComplexFormTypes().ToBlockingEnumerable().Should().ContainEquivalentOf(complexFormEntry);
     }
+
+    [Fact]
+    [Trait("Category", "Integration")]
+    public async Task SyncWithLegacySnapshot_EmptyMorphTypes_DoesNotDuplicate()
+    {
+        var crdtApi = _fixture.CrdtApi;
+        var fwdataApi = _fixture.FwDataApi;
+
+        // First sync: import so both sides have data
+        await _syncService.Import(crdtApi, fwdataApi);
+        var snapshot = await _fixture.RegenerateAndGetSnapshot();
+
+        // Simulate a legacy snapshot by clearing MorphTypes
+        var legacySnapshot = snapshot with { MorphTypes = [] };
+
+        // The CRDT should already have morph types (from seeding in MigrateDb).
+        // Syncing with a legacy snapshot should patch the snapshot and not duplicate morph types.
+        var syncResult = await _syncService.Sync(crdtApi, fwdataApi, legacySnapshot);
+
+        // Verify no duplicates
+        var crdtMorphTypes = await crdtApi.GetMorphTypes().ToArrayAsync();
+        crdtMorphTypes.Should().OnlyHaveUniqueItems(mt => mt.Kind);
+        crdtMorphTypes.Should().NotBeEmpty();
+
+        // Verify no morph-type changes were needed (they were patched from CRDT)
+        syncResult.CrdtChanges.Should().Be(0);
+        syncResult.FwdataChanges.Should().Be(0);
+    }
 }

backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs

@@ -76,6 +76,18 @@ private async Task<SyncResult> SyncOrImportInternal(IMiniLcmApi crdtApi, IMiniLc
         {
             // Repair any missing translation IDs before doing the full sync, so the sync doesn't have to deal with them
             var syncedIdCount = await CrdtRepairs.SyncMissingTranslationIds(projectSnapshot.Entries, fwdata, crdt, dryRun);
+
+            // Patch legacy snapshots that were created before morph-type support.
+            // After seeding, the CRDT has morph-types but the snapshot still has [].
+            // Without this patch, the diff would see all morph-types as "new" and try to re-add them.
+            if (projectSnapshot.MorphTypes.Length == 0)
+            {
+                var currentCrdtMorphTypes = await crdt.GetMorphTypes().ToArrayAsync();
+                if (currentCrdtMorphTypes.Length > 0)
+                {
+                    projectSnapshot = projectSnapshot with { MorphTypes = currentCrdtMorphTypes };
+                }
+            }
         }
 
         var syncResult = projectSnapshot is null

backend/FwLite/LcmCrdt.Tests/Changes/ChangeSerializationTests.cs

@@ -124,6 +124,7 @@ public void CanDeserializeLegacyRegressionData()
     }
 
     [Fact]
+    [Trait("Category", "Verified")]
     public async Task RegressionDataUpToDate()
     {
         var legacyJsonArray = ReadJsonArrayFromFile(GetJsonFilePath("ChangeDeserializationRegressionData.legacy.verified.txt"));

backend/FwLite/LcmCrdt.Tests/Data/FilteringTests.cs

@@ -1,19 +1,20 @@
 using LcmCrdt.Data;
-using MiniLcm.Models;
 
 namespace LcmCrdt.Tests.Data;
 
 public class FilteringTests
 {
     private readonly List<Entry> _entries;
+    private readonly IQueryable<MorphType> _morphTypes;
 
     public FilteringTests()
     {
         _entries =
         [
             new Entry { LexemeForm = { { "en", "123" } }, },
-            new Entry { LexemeForm = { { "en", "456" } }, }
+            new Entry { LexemeForm = { { "en", "456" } }, },
         ];
+        _morphTypes = CanonicalMorphTypes.All.Values.ToArray().AsQueryable();
     }
 
     [Theory]
@@ -36,7 +37,7 @@ public void WhereExemplar_CompiledFilter_ShouldReturnSameResults(string exemplar
     [InlineData("9")]
     public void SearchFilter_CompiledFilter_ShouldReturnSameResults(string query)
     {
-        var expected = _entries.AsQueryable().Where(Filtering.SearchFilter(query)).ToList();
+        var expected = Filtering.SearchFilter(_entries.AsQueryable(), _morphTypes, query).ToList();
 
         var actual = _entries.Where(Filtering.CompiledFilter(query, "en", null)).ToList();
 
@@ -52,9 +53,8 @@ public void CombinedFilter_CompiledFilter_ShouldReturnSameResults(string exempla
     {
         WritingSystemId ws = "en";
 
-        var expected = _entries.AsQueryable()
-            .WhereExemplar(ws, exemplar)
-            .Where(Filtering.SearchFilter(query))
+        var expected = Filtering.SearchFilter(
+                _entries.AsQueryable().WhereExemplar(ws, exemplar), _morphTypes, query)
             .ToList();
 
         var actual = _entries.Where(Filtering.CompiledFilter(query, ws, exemplar)).ToList();