diff --git a/Serval.sln b/Serval.sln index ecf0b237c..ade31e68f 100644 --- a/Serval.sln +++ b/Serval.sln @@ -92,6 +92,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{1DB5E6D1-1 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.ServiceToolkit.Tests", "src\ServiceToolkit\test\SIL.ServiceToolkit.Tests\SIL.ServiceToolkit.Tests.csproj", "{C50ED15A-876D-42BF-980A-388E8C49C78D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.IntegrationTests", "src\Serval\test\Serval.IntegrationTests\Serval.IntegrationTests.csproj", "{5FC2A081-9C4A-4761-BCE1-9753C942D597}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serval.Machine.IntegrationTests", "src\Machine\test\Serval.Machine.IntegrationTests\Serval.Machine.IntegrationTests.csproj", "{D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -194,6 +198,14 @@ Global {C50ED15A-876D-42BF-980A-388E8C49C78D}.Debug|Any CPU.Build.0 = Debug|Any CPU {C50ED15A-876D-42BF-980A-388E8C49C78D}.Release|Any CPU.ActiveCfg = Release|Any CPU {C50ED15A-876D-42BF-980A-388E8C49C78D}.Release|Any CPU.Build.0 = Release|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5FC2A081-9C4A-4761-BCE1-9753C942D597}.Release|Any CPU.Build.0 = Release|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -232,6 +244,8 @@ Global {5E3D2BC3-9A98-4106-A2BF-B1F3641DC6F5} = {3E753B99-7C31-42AC-B02E-012B802F58DB} {1DB5E6D1-17A8-4FF2-B90A-C5DFBEF63126} = {EA69B41C-49EF-4017-A687-44B9DF37FF98} {C50ED15A-876D-42BF-980A-388E8C49C78D} = {1DB5E6D1-17A8-4FF2-B90A-C5DFBEF63126} + {5FC2A081-9C4A-4761-BCE1-9753C942D597} = {3E753B99-7C31-42AC-B02E-012B802F58DB} + {D6C99413-E63C-40EE-8A25-1D3EE78EE5B0} = {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfiguratorExtensions.cs b/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfiguratorExtensions.cs index 6174a93bb..ae089d3a4 100644 --- a/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfiguratorExtensions.cs @@ -6,7 +6,7 @@ public static IMongoDataAccessConfigurator AddRepository( this IMongoDataAccessConfigurator configurator, string collectionName, Action>? mapSetup = null, - Func, Task>? init = null + IReadOnlyList, Task>>? init = null ) where T : IEntity { @@ -16,7 +16,27 @@ public static IMongoDataAccessConfigurator AddRepository( { configurator.Services.Configure(options => { - options.Initializers.Add(database => init(database.GetCollection(collectionName))); + options.Initializers.Add( + async (serviceProvider, database) => + { + using IServiceScope scope = serviceProvider.CreateScope(); + var schemaVersions = scope.ServiceProvider.GetRequiredService>(); + SchemaVersion? schemaVersion = await schemaVersions.GetAsync(s => + s.Collection == collectionName + ); + int currentVersion = schemaVersion?.Version ?? 0; + IMongoCollection collection = database.GetCollection(collectionName); + for (int i = currentVersion + 1; i <= init.Count; i++) + { + await init[i - 1](collection); + await schemaVersions.UpdateAsync( + s => s.Collection == collectionName, + u => u.Set(s => s.Version, i), + upsert: true + ); + } + } + ); }); } diff --git a/src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs b/src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs index 8432a7064..6d77002ca 100644 --- a/src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs +++ b/src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs @@ -30,6 +30,17 @@ Action configure new ObjectRefConvention() ); + // Configure conventions for schema_versions + DataAccessClassMap.RegisterConventions( + "SIL.DataAccess.Models", + new StringIdStoredAsObjectIdConvention(), + new CamelCaseElementNameConvention(), + new EnumRepresentationConvention(BsonType.String), + new IgnoreExtraElementsConvention(true), + new IgnoreIfNullConvention(true), + new ObjectRefConvention() + ); + services.Configure(options => options.Url = new MongoUrl(connectionString)); services.TryAddTransient(); services.TryAddSingleton(sp => @@ -45,7 +56,23 @@ Action configure services.TryAddScoped(); services.TryAddScoped(sp => sp.GetRequiredService()); services.AddHostedService(); - configure(new MongoDataAccessConfigurator(services)); + var configurator = new MongoDataAccessConfigurator(services); + + // Configure the schema_versions repository + configurator.AddRepository( + "schema_versions", + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.Collection) + ) + ), + ] + ); + + configure(configurator); return services; } } diff --git a/src/DataAccess/src/SIL.DataAccess/Models/SchemaVersion.cs b/src/DataAccess/src/SIL.DataAccess/Models/SchemaVersion.cs new file mode 100644 index 000000000..d6d09fe07 --- /dev/null +++ b/src/DataAccess/src/SIL.DataAccess/Models/SchemaVersion.cs @@ -0,0 +1,9 @@ +namespace SIL.DataAccess.Models; + +public class SchemaVersion : IEntity +{ + public required string Id { get; set; } + public int Revision { get; set; } + public required string Collection { get; set; } + public int Version { get; set; } +} diff --git a/src/DataAccess/src/SIL.DataAccess/MongoDataAccessInitializeService.cs b/src/DataAccess/src/SIL.DataAccess/MongoDataAccessInitializeService.cs index 8fbd8522e..e175ce7bd 100644 --- a/src/DataAccess/src/SIL.DataAccess/MongoDataAccessInitializeService.cs +++ b/src/DataAccess/src/SIL.DataAccess/MongoDataAccessInitializeService.cs @@ -1,19 +1,16 @@ namespace SIL.DataAccess; -public class MongoDataAccessInitializeService(IMongoDatabase database, IOptions options) - : IHostedService +public class MongoDataAccessInitializeService( + IServiceProvider provider, + IMongoDatabase database, + IOptions options +) : IHostedService { - private readonly IMongoDatabase _database = database; - private readonly IOptions _options = options; - public async Task StartAsync(CancellationToken cancellationToken) { - foreach (Func initializer in _options.Value.Initializers) - await initializer(_database).ConfigureAwait(false); + foreach (Func initializer in options.Value.Initializers) + await initializer(provider, database).ConfigureAwait(false); } - public Task StopAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; } diff --git a/src/DataAccess/src/SIL.DataAccess/MongoDataAccessOptions.cs b/src/DataAccess/src/SIL.DataAccess/MongoDataAccessOptions.cs index d1013470d..d0b35c4d1 100644 --- a/src/DataAccess/src/SIL.DataAccess/MongoDataAccessOptions.cs +++ b/src/DataAccess/src/SIL.DataAccess/MongoDataAccessOptions.cs @@ -3,5 +3,5 @@ public class MongoDataAccessOptions { public MongoUrl Url { get; set; } = new MongoUrl("mongodb://localhost:27017"); - public IList> Initializers { get; } = new List>(); + public IList> Initializers { get; } = []; } diff --git a/src/DataAccess/src/SIL.DataAccess/Usings.cs b/src/DataAccess/src/SIL.DataAccess/Usings.cs index 98d00d85e..f1f9e4adf 100644 --- a/src/DataAccess/src/SIL.DataAccess/Usings.cs +++ b/src/DataAccess/src/SIL.DataAccess/Usings.cs @@ -17,3 +17,4 @@ global using Newtonsoft.Json.Serialization; global using Nito.AsyncEx; global using SIL.DataAccess; +global using SIL.DataAccess.Models; diff --git a/src/DataAccess/test/SIL.DataAccess.Tests/SIL.DataAccess.Tests.csproj b/src/DataAccess/test/SIL.DataAccess.Tests/SIL.DataAccess.Tests.csproj index b3984b602..873b71e30 100644 --- a/src/DataAccess/test/SIL.DataAccess.Tests/SIL.DataAccess.Tests.csproj +++ b/src/DataAccess/test/SIL.DataAccess.Tests/SIL.DataAccess.Tests.csproj @@ -13,19 +13,19 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index 52a15b9d9..ae000d3d8 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -215,45 +215,54 @@ public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder) { o.AddRepository( "translation_engines", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.EngineId) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) - ) - ); - } + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.EngineId) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) + ) + ), + ] ); o.AddRepository( "word_alignment_engines", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.EngineId) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) - ) - ); - } + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.EngineId) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => + e.CurrentBuild!.BuildJobRunner + ) + ) + ), + ] ); o.AddRepository("locks"); o.AddRepository( "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) + ) + ), + ] ); } ); diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs index 5fece454f..392e33740 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs @@ -1,6 +1,6 @@ namespace Microsoft.Extensions.DependencyInjection; -internal class MachineBuilder(IServiceCollection services, IConfiguration configuration) : IMachineBuilder +public class MachineBuilder(IServiceCollection services, IConfiguration configuration) : IMachineBuilder { public IServiceCollection Services { get; } = services; public IConfiguration Configuration { get; } = configuration; diff --git a/src/Machine/src/Serval.Machine.Shared/Usings.cs b/src/Machine/src/Serval.Machine.Shared/Usings.cs index 7e6db4fa3..405e8a4dc 100644 --- a/src/Machine/src/Serval.Machine.Shared/Usings.cs +++ b/src/Machine/src/Serval.Machine.Shared/Usings.cs @@ -1,8 +1,6 @@ global using System.Collections.Concurrent; -global using System.Collections.Generic; global using System.Collections.Immutable; global using System.ComponentModel; -global using System.Data; global using System.Diagnostics; global using System.Formats.Tar; global using System.Globalization; @@ -24,7 +22,6 @@ global using Amazon.S3; global using Amazon.S3.Model; global using Bugsnag.AspNet.Core; -global using CommunityToolkit.HighPerformance; global using Grpc.Core; global using Grpc.Core.Interceptors; global using Grpc.Net.Client.Configuration; @@ -43,7 +40,6 @@ global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using MongoDB.Driver; -global using MongoDB.Driver.Linq; global using Nito.AsyncEx; global using Nito.AsyncEx.Synchronous; global using Polly; diff --git a/src/Machine/test/Serval.Machine.IntegrationTests/Serval.Machine.IntegrationTests.csproj b/src/Machine/test/Serval.Machine.IntegrationTests/Serval.Machine.IntegrationTests.csproj new file mode 100644 index 000000000..6b368a8af --- /dev/null +++ b/src/Machine/test/Serval.Machine.IntegrationTests/Serval.Machine.IntegrationTests.csproj @@ -0,0 +1,38 @@ + + + + net10.0 + enable + enable + false + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + diff --git a/src/Machine/test/Serval.Machine.IntegrationTests/ServalMachineSharedTests.cs b/src/Machine/test/Serval.Machine.IntegrationTests/ServalMachineSharedTests.cs new file mode 100644 index 000000000..31edab2ad --- /dev/null +++ b/src/Machine/test/Serval.Machine.IntegrationTests/ServalMachineSharedTests.cs @@ -0,0 +1,35 @@ +namespace Serval.Machine.IntegrationTests; + +[TestFixture] +[Category("Integration")] +public class ServalMachineSharedTests +{ + private TestEnvironment _env; + + [SetUp] + public void SetUp() + { + _env = new TestEnvironment(); + } + + [Test] + public async Task InitializesRepositories() + { + // Setup + IMachineBuilder machineBuilder = new MachineBuilder(_env.Services, _env.Configuration); + machineBuilder.AddMongoDataAccess(); + + // SUT + await _env.InitializeDatabaseAsync(); + + // Verify schema versioning + SchemaVersion? schemaVersion = await _env.SchemaVersions!.GetAsync(s => s.Collection == "schema_versions"); + Assert.That(schemaVersion!.Version, Is.EqualTo(1)); + } + + [TearDown] + public void TearDown() + { + _env.Dispose(); + } +} diff --git a/src/Machine/test/Serval.Machine.IntegrationTests/TestEnvironment.cs b/src/Machine/test/Serval.Machine.IntegrationTests/TestEnvironment.cs new file mode 100644 index 000000000..5b5bc6788 --- /dev/null +++ b/src/Machine/test/Serval.Machine.IntegrationTests/TestEnvironment.cs @@ -0,0 +1,67 @@ +namespace Serval.Machine.IntegrationTests; + +public class TestEnvironment : DisposableBase +{ + public const string DatabaseName = "serval_test"; + public readonly MongoClient MongoClient = new(); + public IRepository? SchemaVersions { get; private set; } + public readonly ServiceCollection Services = []; + public readonly IConfiguration Configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["ConnectionStrings:Mongo"] = $"mongodb://localhost:27017/{DatabaseName}", + } + ) + .Build(); + + public TestEnvironment() + { + Services.AddLogging(); + Services.AddSingleton(); + ResetDatabase(); + } + + public async Task InitializeDatabaseAsync() + { + ServiceProvider provider = Services.BuildServiceProvider(); + + SchemaVersions = provider.GetRequiredService>(); + MongoDataAccessInitializeService mongoDataAccessInitializeService = + provider.GetRequiredService(); + await mongoDataAccessInitializeService.StartAsync(CancellationToken.None); + } + + public Task GetDocumentAsync(string collection, ObjectId objectId) => + MongoClient + .GetDatabase(DatabaseName) + .GetCollection(collection) + .Find(Builders.Filter.Eq("_id", objectId)) + .FirstOrDefaultAsync(); + + public Task InsertDocumentAsync(string collection, BsonDocument document) => + MongoClient.GetDatabase(DatabaseName).GetCollection(collection).InsertOneAsync(document); + + public Task SetupSchemaAsync(string collection, int version) => + InsertDocumentAsync( + "schema_versions", + new BsonDocument + { + { "_id", ObjectId.GenerateNewId() }, + { "collection", collection }, + { "version", version }, + { "revision", 1 }, + } + ); + + protected override void DisposeManagedResources() + { + ResetDatabase(); + MongoClient.Dispose(); + } + + private void ResetDatabase() + { + MongoClient.DropDatabase(DatabaseName); + } +} diff --git a/src/Machine/test/Serval.Machine.IntegrationTests/Usings.cs b/src/Machine/test/Serval.Machine.IntegrationTests/Usings.cs new file mode 100644 index 000000000..e56b93a61 --- /dev/null +++ b/src/Machine/test/Serval.Machine.IntegrationTests/Usings.cs @@ -0,0 +1,8 @@ +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using MongoDB.Bson; +global using MongoDB.Driver; +global using NUnit.Framework; +global using SIL.DataAccess; +global using SIL.DataAccess.Models; +global using SIL.ObjectModel; diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj b/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj index 30dd302a4..9af2ff372 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj @@ -13,25 +13,25 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Serval/src/Serval.DataFiles/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.DataFiles/Configuration/IMongoDataAccessConfiguratorExtensions.cs index 37a1e4f0e..f3fc035de 100644 --- a/src/Serval/src/Serval.DataFiles/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.DataFiles/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -8,32 +8,40 @@ public static IMongoDataAccessConfigurator AddDataFilesRepositories(this IMongoD { configurator.AddRepository( "data_files.files", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(p => p.Owner)) - ) + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(p => p.Owner)) + ), + ] ); configurator.AddRepository( "data_files.deleted_files", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(p => p.DeletedAt)) - ) + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(p => p.DeletedAt)) + ), + ] ); configurator.AddRepository( "corpora.corpus", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(p => p.Owner)) - ); + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(p => p.Owner)) + ), // migrate by adding Name field - await c.UpdateManyAsync( - Builders.Filter.Exists(b => b.Name, false), - Builders.Update.Set(b => b.Name, null) - ); - } + c => + c.UpdateManyAsync( + Builders.Filter.Exists(b => b.Name, false), + Builders.Update.Set(b => b.Name, null) + ), + ] ); return configurator; } diff --git a/src/Serval/src/Serval.Shared/Configuration/ServalBuilder.cs b/src/Serval/src/Serval.Shared/Configuration/ServalBuilder.cs index 48c5123d3..f2f25f4f6 100644 --- a/src/Serval/src/Serval.Shared/Configuration/ServalBuilder.cs +++ b/src/Serval/src/Serval.Shared/Configuration/ServalBuilder.cs @@ -1,6 +1,6 @@ namespace Microsoft.Extensions.DependencyInjection; -internal class ServalBuilder(IServiceCollection services, IConfiguration configuration) : IServalBuilder +public class ServalBuilder(IServiceCollection services, IConfiguration configuration) : IServalBuilder { public IServiceCollection Services { get; } = services; public IConfiguration Configuration { get; } = configuration; diff --git a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs index 40695b5df..5c6dbed3e 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -11,93 +11,112 @@ this IMongoDataAccessConfigurator configurator { configurator.AddRepository( "translation.engines", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.Owner)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.DateCreated)) - ); + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.Owner)) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.DateCreated)) + ), // migrate to new ParallelCorpora scheme by adding ParallelCorpora to existing engines - await c.UpdateManyAsync( - Builders.Filter.Exists(e => e.ParallelCorpora, false), - Builders.Update.Set(e => e.ParallelCorpora, new List()) - ); - } + c => + c.UpdateManyAsync( + Builders.Filter.Exists(e => e.ParallelCorpora, false), + Builders.Update.Set(e => e.ParallelCorpora, []) + ), + ] ); configurator.AddRepository( "translation.builds", - init: static async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.Owner)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.EngineRef)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.DateCreated)) - ); + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.Owner)) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.EngineRef)) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.DateCreated)) + ), // migrate by adding ExecutionData field - await c.UpdateManyAsync( - Builders.Filter.Exists(b => b.ExecutionData, false), - Builders.Update.Set(b => b.ExecutionData, new ExecutionData()) - ); + c => + c.UpdateManyAsync( + Builders.Filter.Exists(b => b.ExecutionData, false), + Builders.Update.Set(b => b.ExecutionData, new ExecutionData()) + ), // migrate the percentCompleted field to the progress field - await c.UpdateManyAsync( - Builders.Filter.And( - Builders.Filter.Exists("percentCompleted"), - Builders.Filter.Exists(b => b.Progress, false) + c => + c.UpdateManyAsync( + Builders.Filter.And( + Builders.Filter.Exists("percentCompleted"), + Builders.Filter.Exists(b => b.Progress, false) + ), + new BsonDocument("$rename", new BsonDocument("percentCompleted", "progress")) ), - new BsonDocument("$rename", new BsonDocument("percentCompleted", "progress")) - ); // migrate by duplicating the owner field from build - await c.Aggregate() - .Match(Builders.Filter.Exists(b => b.Owner, false)) - .Lookup("translation.engines", "engineRef", "_id", "engine") - .Unwind("engine", new AggregateUnwindOptions { PreserveNullAndEmptyArrays = true }) - .AppendStage(new BsonDocument("$set", new BsonDocument("owner", "$engine.owner"))) - .AppendStage(new BsonDocument("$unset", "engine")) - .Merge(c, new MergeStageOptions { WhenMatched = MergeStageWhenMatched.Replace }) - .ToListAsync(); - await MongoMigrations.MigrateTargetQuoteConvention(c); - } + c => + c.Aggregate() + .Match(Builders.Filter.Exists(b => b.Owner, false)) + .Lookup("translation.engines", "engineRef", "_id", "engine") + .Unwind( + "engine", + new AggregateUnwindOptions { PreserveNullAndEmptyArrays = true } + ) + .AppendStage(new BsonDocument("$set", new BsonDocument("owner", "$engine.owner"))) + .AppendStage(new BsonDocument("$unset", "engine")) + .Merge(c, new MergeStageOptions { WhenMatched = MergeStageWhenMatched.Replace }) + .ToListAsync(), + MongoMigrations.MigrateTargetQuoteConvention, + ] ); configurator.AddRepository( "translation.pretranslations", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(pt => pt.ModelRevision) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(pt => pt.CorpusRef) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.TextId)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders - .IndexKeys.Ascending(pt => pt.EngineRef) - .Ascending(pt => pt.ModelRevision) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders - .IndexKeys.Ascending(pt => pt.EngineRef) - .Ascending(pt => pt.CorpusRef) - .Ascending(pt => pt.ModelRevision) - .Ascending(pt => pt.TextId) - ) - ); - } + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.ModelRevision) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.CorpusRef) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.TextId) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders + .IndexKeys.Ascending(pt => pt.EngineRef) + .Ascending(pt => pt.ModelRevision) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders + .IndexKeys.Ascending(pt => pt.EngineRef) + .Ascending(pt => pt.CorpusRef) + .Ascending(pt => pt.ModelRevision) + .Ascending(pt => pt.TextId) + ) + ), + ] ); return configurator; } diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs index 2551de844..2d3084fba 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -8,15 +8,17 @@ public static IMongoDataAccessConfigurator AddWebhooksRepositories(this IMongoDa { configurator.AddRepository( "webhooks.hooks", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Owner)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Events)) - ); - } + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Owner)) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Events)) + ), + ] ); return configurator; } diff --git a/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs index a36d34779..8c92b5ccb 100644 --- a/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.WordAlignment/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -11,68 +11,82 @@ this IMongoDataAccessConfigurator configurator { configurator.AddRepository( "word_alignment.engines", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.Owner)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.DateCreated)) - ); - } + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.Owner)) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.DateCreated)) + ), + ] ); configurator.AddRepository( "word_alignment.builds", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.EngineRef)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.DateCreated)) - ); + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.EngineRef)) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(b => b.DateCreated)) + ), // migrate the percentCompleted field to the progress field - await c.UpdateManyAsync( - Builders.Filter.And( - Builders.Filter.Exists("percentCompleted"), - Builders.Filter.Exists(b => b.Progress, false) + c => + c.UpdateManyAsync( + Builders.Filter.And( + Builders.Filter.Exists("percentCompleted"), + Builders.Filter.Exists(b => b.Progress, false) + ), + new BsonDocument("$rename", new BsonDocument("percentCompleted", "progress")) ), - new BsonDocument("$rename", new BsonDocument("percentCompleted", "progress")) - ); - } + ] ); configurator.AddRepository( "word_alignment.word_alignments", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(pt => pt.ModelRevision) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.CorpusRef)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.TextId)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders - .IndexKeys.Ascending(pt => pt.EngineRef) - .Ascending(pt => pt.ModelRevision) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders - .IndexKeys.Ascending(pt => pt.EngineRef) - .Ascending(pt => pt.CorpusRef) - .Ascending(pt => pt.ModelRevision) - .Ascending(pt => pt.TextId) - ) - ); - } + init: + [ + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.ModelRevision) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.CorpusRef) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.TextId) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders + .IndexKeys.Ascending(pt => pt.EngineRef) + .Ascending(pt => pt.ModelRevision) + ) + ), + c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders + .IndexKeys.Ascending(pt => pt.EngineRef) + .Ascending(pt => pt.CorpusRef) + .Ascending(pt => pt.ModelRevision) + .Ascending(pt => pt.TextId) + ) + ), + ] ); return configurator; } diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/Serval.ApiServer.IntegrationTests.csproj b/src/Serval/test/Serval.ApiServer.IntegrationTests/Serval.ApiServer.IntegrationTests.csproj index e598c1783..2cfc84358 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/Serval.ApiServer.IntegrationTests.csproj +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/Serval.ApiServer.IntegrationTests.csproj @@ -17,20 +17,20 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Serval/test/Serval.DataFiles.Tests/Serval.DataFiles.Tests.csproj b/src/Serval/test/Serval.DataFiles.Tests/Serval.DataFiles.Tests.csproj index 6694ffc8e..4b68dab9b 100644 --- a/src/Serval/test/Serval.DataFiles.Tests/Serval.DataFiles.Tests.csproj +++ b/src/Serval/test/Serval.DataFiles.Tests/Serval.DataFiles.Tests.csproj @@ -13,19 +13,19 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Serval/test/Serval.E2ETests/Serval.E2ETests.csproj b/src/Serval/test/Serval.E2ETests/Serval.E2ETests.csproj index 2c4918aa1..901cc6c95 100644 --- a/src/Serval/test/Serval.E2ETests/Serval.E2ETests.csproj +++ b/src/Serval/test/Serval.E2ETests/Serval.E2ETests.csproj @@ -11,9 +11,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Serval/test/Serval.IntegrationTests/Serval.IntegrationTests.csproj b/src/Serval/test/Serval.IntegrationTests/Serval.IntegrationTests.csproj new file mode 100644 index 000000000..b8076a89f --- /dev/null +++ b/src/Serval/test/Serval.IntegrationTests/Serval.IntegrationTests.csproj @@ -0,0 +1,43 @@ + + + + net10.0 + enable + enable + false + + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + diff --git a/src/Serval/test/Serval.IntegrationTests/ServalSharedTests.cs b/src/Serval/test/Serval.IntegrationTests/ServalSharedTests.cs new file mode 100644 index 000000000..a0ddd8678 --- /dev/null +++ b/src/Serval/test/Serval.IntegrationTests/ServalSharedTests.cs @@ -0,0 +1,73 @@ +namespace Serval.IntegrationTests; + +[TestFixture] +[Category("Integration")] +public class ServalSharedTests +{ + private TestEnvironment _env; + + [SetUp] + public void SetUp() + { + _env = new TestEnvironment(); + } + + [Test] + public async Task InitializesRepositories() + { + // Setup + IServalBuilder servalBuilder = new ServalBuilder(_env.Services, _env.Configuration); + servalBuilder.AddMongoDataAccess(cfg => + { + cfg.AddTranslationRepositories(); + cfg.AddWordAlignmentRepositories(); + cfg.AddDataFilesRepositories(); + cfg.AddWebhooksRepositories(); + }); + + // SUT + await _env.InitializeDatabaseAsync(); + + // Verify schema versioning + SchemaVersion? schemaVersion = await _env.SchemaVersions!.GetAsync(s => s.Collection == "schema_versions"); + Assert.That(schemaVersion!.Version, Is.EqualTo(1)); + } + + [Test] + public async Task Migrates_TranslationEngines_ParallelCorpora() + { + // Setup + IServalBuilder servalBuilder = new ServalBuilder(_env.Services, _env.Configuration); + servalBuilder.AddMongoDataAccess(cfg => + { + cfg.AddTranslationRepositories(); + }); + + // Populate pre-migration-data + await _env.SetupSchemaAsync("translation.engines", 2); + var objectId = ObjectId.GenerateNewId(); + await _env.InsertDocumentAsync("translation.engines", new BsonDocument { { "_id", objectId } }); + + // SUT + await _env.InitializeDatabaseAsync(); + + // Verify schema version change + SchemaVersion? schemaVersion = await _env.SchemaVersions!.GetAsync(s => s.Collection == "translation.engines"); + Assert.That(schemaVersion!.Version, Is.GreaterThan(2)); + + // Verify migration + BsonDocument? document = await _env.GetDocumentAsync("translation.engines", objectId); + Assert.Multiple(() => + { + Assert.That(document.Contains("parallelCorpora"), Is.True); + Assert.That(document["parallelCorpora"].IsBsonArray, Is.True); + Assert.That(document["parallelCorpora"].AsBsonArray, Is.Empty); + }); + } + + [TearDown] + public void TearDown() + { + _env.Dispose(); + } +} diff --git a/src/Serval/test/Serval.IntegrationTests/TestEnvironment.cs b/src/Serval/test/Serval.IntegrationTests/TestEnvironment.cs new file mode 100644 index 000000000..511eec1e9 --- /dev/null +++ b/src/Serval/test/Serval.IntegrationTests/TestEnvironment.cs @@ -0,0 +1,67 @@ +namespace Serval.IntegrationTests; + +public class TestEnvironment : DisposableBase +{ + public const string DatabaseName = "serval_test"; + public readonly MongoClient MongoClient = new(); + public IRepository? SchemaVersions { get; private set; } + public readonly ServiceCollection Services = []; + public readonly IConfiguration Configuration = new ConfigurationBuilder() + .AddInMemoryCollection( + new Dictionary + { + ["ConnectionStrings:Mongo"] = $"mongodb://localhost:27017/{DatabaseName}", + } + ) + .Build(); + + public TestEnvironment() + { + Services.AddLogging(); + Services.AddSingleton(); + ResetDatabase(); + } + + public async Task InitializeDatabaseAsync() + { + ServiceProvider provider = Services.BuildServiceProvider(); + + SchemaVersions = provider.GetRequiredService>(); + MongoDataAccessInitializeService mongoDataAccessInitializeService = + provider.GetRequiredService(); + await mongoDataAccessInitializeService.StartAsync(CancellationToken.None); + } + + public Task GetDocumentAsync(string collection, ObjectId objectId) => + MongoClient + .GetDatabase(DatabaseName) + .GetCollection(collection) + .Find(Builders.Filter.Eq("_id", objectId)) + .FirstOrDefaultAsync(); + + public Task InsertDocumentAsync(string collection, BsonDocument document) => + MongoClient.GetDatabase(DatabaseName).GetCollection(collection).InsertOneAsync(document); + + public Task SetupSchemaAsync(string collection, int version) => + InsertDocumentAsync( + "schema_versions", + new BsonDocument + { + { "_id", ObjectId.GenerateNewId() }, + { "collection", collection }, + { "version", version }, + { "revision", 1 }, + } + ); + + protected override void DisposeManagedResources() + { + ResetDatabase(); + MongoClient.Dispose(); + } + + private void ResetDatabase() + { + MongoClient.DropDatabase(DatabaseName); + } +} diff --git a/src/Serval/test/Serval.IntegrationTests/Usings.cs b/src/Serval/test/Serval.IntegrationTests/Usings.cs new file mode 100644 index 000000000..e56b93a61 --- /dev/null +++ b/src/Serval/test/Serval.IntegrationTests/Usings.cs @@ -0,0 +1,8 @@ +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using MongoDB.Bson; +global using MongoDB.Driver; +global using NUnit.Framework; +global using SIL.DataAccess; +global using SIL.DataAccess.Models; +global using SIL.ObjectModel; diff --git a/src/Serval/test/Serval.Shared.Tests/Serval.Shared.Tests.csproj b/src/Serval/test/Serval.Shared.Tests/Serval.Shared.Tests.csproj index 36c358e63..a08beb8a9 100644 --- a/src/Serval/test/Serval.Shared.Tests/Serval.Shared.Tests.csproj +++ b/src/Serval/test/Serval.Shared.Tests/Serval.Shared.Tests.csproj @@ -14,19 +14,19 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj b/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj index 48d9ed5f2..93e60253c 100644 --- a/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj +++ b/src/Serval/test/Serval.Translation.Tests/Serval.Translation.Tests.csproj @@ -13,19 +13,19 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Serval/test/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj b/src/Serval/test/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj index 46b343c32..3cd1df699 100644 --- a/src/Serval/test/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj +++ b/src/Serval/test/Serval.Webhooks.Tests/Serval.Webhooks.Tests.csproj @@ -12,19 +12,19 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Serval/test/Serval.WordAlignment.Tests/Serval.WordAlignment.Tests.csproj b/src/Serval/test/Serval.WordAlignment.Tests/Serval.WordAlignment.Tests.csproj index 6c76ef88c..a30fd14b5 100644 --- a/src/Serval/test/Serval.WordAlignment.Tests/Serval.WordAlignment.Tests.csproj +++ b/src/Serval/test/Serval.WordAlignment.Tests/Serval.WordAlignment.Tests.csproj @@ -13,19 +13,19 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/SIL.ServiceToolkit.Tests.csproj b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/SIL.ServiceToolkit.Tests.csproj index 86083f735..19723403e 100644 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/SIL.ServiceToolkit.Tests.csproj +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/SIL.ServiceToolkit.Tests.csproj @@ -8,19 +8,19 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive