Skip to content

Commit d247ecc

Browse files
pmachapmanpmachapman
committed
Added schema versioning for database migrations
1 parent 712381a commit d247ecc

11 files changed

Lines changed: 310 additions & 204 deletions

File tree

‎src/DataAccess/src/SIL.DataAccess/IMongoDataAccessConfiguratorExtensions.cs‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ public static IMongoDataAccessConfigurator AddRepository<T>(
66
this IMongoDataAccessConfigurator configurator,
77
string collectionName,
88
Action<BsonClassMap<T>>? mapSetup = null,
9-
Func<IMongoCollection<T>, Task>? init = null
9+
Func<IMongoCollection<T>, Task>[]? init = null
1010
)
1111
where T : IEntity
1212
{
@@ -16,7 +16,27 @@ public static IMongoDataAccessConfigurator AddRepository<T>(
1616
{
1717
configurator.Services.Configure<MongoDataAccessOptions>(options =>
1818
{
19-
options.Initializers.Add(database => init(database.GetCollection<T>(collectionName)));
19+
options.Initializers.Add(
20+
async (serviceProvider, database) =>
21+
{
22+
using IServiceScope scope = serviceProvider.CreateScope();
23+
var schemaVersions = scope.ServiceProvider.GetRequiredService<IRepository<SchemaVersion>>();
24+
SchemaVersion? schemaVersion = await schemaVersions.GetAsync(s =>
25+
s.Collection == collectionName
26+
);
27+
int currentVersion = schemaVersion?.Version ?? 0;
28+
IMongoCollection<T> collection = database.GetCollection<T>(collectionName);
29+
for (int i = currentVersion + 1; i <= init.Length; i++)
30+
{
31+
await init[i - 1](collection);
32+
await schemaVersions.UpdateAsync(
33+
s => s.Collection == collectionName,
34+
u => u.Set(s => s.Version, i),
35+
upsert: true
36+
);
37+
}
38+
}
39+
);
2040
});
2141
}
2242

‎src/DataAccess/src/SIL.DataAccess/IServiceCollectionExtensions.cs‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,17 @@ Action<IMongoDataAccessConfigurator> configure
3030
new ObjectRefConvention()
3131
);
3232

33+
// Configure conventions for schema_versions
34+
DataAccessClassMap.RegisterConventions(
35+
"SIL.DataAccess.Models",
36+
new StringIdStoredAsObjectIdConvention(),
37+
new CamelCaseElementNameConvention(),
38+
new EnumRepresentationConvention(BsonType.String),
39+
new IgnoreExtraElementsConvention(true),
40+
new IgnoreIfNullConvention(true),
41+
new ObjectRefConvention()
42+
);
43+
3344
services.Configure<MongoDataAccessOptions>(options => options.Url = new MongoUrl(connectionString));
3445
services.TryAddTransient<SIL.DataAccess.IIdGenerator, ObjectIdGenerator>();
3546
services.TryAddSingleton<IMongoClient>(sp =>
@@ -45,7 +56,23 @@ Action<IMongoDataAccessConfigurator> configure
4556
services.TryAddScoped<IMongoDataAccessContext, MongoDataAccessContext>();
4657
services.TryAddScoped<IDataAccessContext>(sp => sp.GetRequiredService<IMongoDataAccessContext>());
4758
services.AddHostedService<MongoDataAccessInitializeService>();
48-
configure(new MongoDataAccessConfigurator(services));
59+
var configurator = new MongoDataAccessConfigurator(services);
60+
61+
// Configure the schema_versions repository
62+
configurator.AddRepository<SchemaVersion>(
63+
"schema_versions",
64+
init:
65+
[
66+
c =>
67+
c.Indexes.CreateOrUpdateAsync(
68+
new CreateIndexModel<SchemaVersion>(
69+
Builders<SchemaVersion>.IndexKeys.Ascending(p => p.Collection)
70+
)
71+
),
72+
]
73+
);
74+
75+
configure(configurator);
4976
return services;
5077
}
5178
}

‎src/DataAccess/src/SIL.DataAccess/Models/SchemaVersion.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
namespace SIL.DataAccess.Models;
2+
3+
public class SchemaVersion : IEntity
4+
{
5+
public required string Id { get; set; }
6+
public int Revision { get; set; }
7+
public required string Collection { get; set; }
8+
public int Version { get; set; }
9+
}

‎src/DataAccess/src/SIL.DataAccess/MongoDataAccessInitializeService.cs‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,16 @@
11
namespace SIL.DataAccess;
22

3-
public class MongoDataAccessInitializeService(IMongoDatabase database, IOptions<MongoDataAccessOptions> options)
4-
: IHostedService
3+
public class MongoDataAccessInitializeService(
4+
IServiceProvider provider,
5+
IMongoDatabase database,
6+
IOptions<MongoDataAccessOptions> options
7+
) : IHostedService
58
{
6-
private readonly IMongoDatabase _database = database;
7-
private readonly IOptions<MongoDataAccessOptions> _options = options;
8-
99
public async Task StartAsync(CancellationToken cancellationToken)
1010
{
11-
foreach (Func<IMongoDatabase, Task> initializer in _options.Value.Initializers)
12-
await initializer(_database).ConfigureAwait(false);
11+
foreach (Func<IServiceProvider, IMongoDatabase, Task> initializer in options.Value.Initializers)
12+
await initializer(provider, database).ConfigureAwait(false);
1313
}
1414

15-
public Task StopAsync(CancellationToken cancellationToken)
16-
{
17-
return Task.CompletedTask;
18-
}
15+
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
1916
}

‎src/DataAccess/src/SIL.DataAccess/MongoDataAccessOptions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,5 @@
33
public class MongoDataAccessOptions
44
{
55
public MongoUrl Url { get; set; } = new MongoUrl("mongodb://localhost:27017");
6-
public IList<Func<IMongoDatabase, Task>> Initializers { get; } = new List<Func<IMongoDatabase, Task>>();
6+
public IList<Func<IServiceProvider, IMongoDatabase, Task>> Initializers { get; } = [];
77
}

‎src/DataAccess/src/SIL.DataAccess/Usings.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@
1717
global using Newtonsoft.Json.Serialization;
1818
global using Nito.AsyncEx;
1919
global using SIL.DataAccess;
20+
global using SIL.DataAccess.Models;

‎src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs‎

Lines changed: 41 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -215,45 +215,54 @@ public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder)
215215
{
216216
o.AddRepository<TranslationEngine>(
217217
"translation_engines",
218-
init: async c =>
219-
{
220-
await c.Indexes.CreateOrUpdateAsync(
221-
new CreateIndexModel<TranslationEngine>(
222-
Builders<TranslationEngine>.IndexKeys.Ascending(e => e.EngineId)
223-
)
224-
);
225-
await c.Indexes.CreateOrUpdateAsync(
226-
new CreateIndexModel<TranslationEngine>(
227-
Builders<TranslationEngine>.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner)
228-
)
229-
);
230-
}
218+
init:
219+
[
220+
c =>
221+
c.Indexes.CreateOrUpdateAsync(
222+
new CreateIndexModel<TranslationEngine>(
223+
Builders<TranslationEngine>.IndexKeys.Ascending(e => e.EngineId)
224+
)
225+
),
226+
c =>
227+
c.Indexes.CreateOrUpdateAsync(
228+
new CreateIndexModel<TranslationEngine>(
229+
Builders<TranslationEngine>.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner)
230+
)
231+
),
232+
]
231233
);
232234
o.AddRepository<WordAlignmentEngine>(
233235
"word_alignment_engines",
234-
init: async c =>
235-
{
236-
await c.Indexes.CreateOrUpdateAsync(
237-
new CreateIndexModel<WordAlignmentEngine>(
238-
Builders<WordAlignmentEngine>.IndexKeys.Ascending(e => e.EngineId)
239-
)
240-
);
241-
await c.Indexes.CreateOrUpdateAsync(
242-
new CreateIndexModel<WordAlignmentEngine>(
243-
Builders<WordAlignmentEngine>.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner)
244-
)
245-
);
246-
}
236+
init:
237+
[
238+
c =>
239+
c.Indexes.CreateOrUpdateAsync(
240+
new CreateIndexModel<WordAlignmentEngine>(
241+
Builders<WordAlignmentEngine>.IndexKeys.Ascending(e => e.EngineId)
242+
)
243+
),
244+
c =>
245+
c.Indexes.CreateOrUpdateAsync(
246+
new CreateIndexModel<WordAlignmentEngine>(
247+
Builders<WordAlignmentEngine>.IndexKeys.Ascending(e =>
248+
e.CurrentBuild!.BuildJobRunner
249+
)
250+
)
251+
),
252+
]
247253
);
248254
o.AddRepository<RWLock>("locks");
249255
o.AddRepository<TrainSegmentPair>(
250256
"train_segment_pairs",
251-
init: c =>
252-
c.Indexes.CreateOrUpdateAsync(
253-
new CreateIndexModel<TrainSegmentPair>(
254-
Builders<TrainSegmentPair>.IndexKeys.Ascending(p => p.TranslationEngineRef)
255-
)
256-
)
257+
init:
258+
[
259+
c =>
260+
c.Indexes.CreateOrUpdateAsync(
261+
new CreateIndexModel<TrainSegmentPair>(
262+
Builders<TrainSegmentPair>.IndexKeys.Ascending(p => p.TranslationEngineRef)
263+
)
264+
),
265+
]
257266
);
258267
}
259268
);

‎src/Serval/src/Serval.DataFiles/Configuration/IMongoDataAccessConfiguratorExtensions.cs‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,32 +8,40 @@ public static IMongoDataAccessConfigurator AddDataFilesRepositories(this IMongoD
88
{
99
configurator.AddRepository<DataFile>(
1010
"data_files.files",
11-
init: c =>
12-
c.Indexes.CreateOrUpdateAsync(
13-
new CreateIndexModel<DataFile>(Builders<DataFile>.IndexKeys.Ascending(p => p.Owner))
14-
)
11+
init:
12+
[
13+
c =>
14+
c.Indexes.CreateOrUpdateAsync(
15+
new CreateIndexModel<DataFile>(Builders<DataFile>.IndexKeys.Ascending(p => p.Owner))
16+
),
17+
]
1518
);
1619

1720
configurator.AddRepository<DeletedFile>(
1821
"data_files.deleted_files",
19-
init: c =>
20-
c.Indexes.CreateOrUpdateAsync(
21-
new CreateIndexModel<DeletedFile>(Builders<DeletedFile>.IndexKeys.Ascending(p => p.DeletedAt))
22-
)
22+
init:
23+
[
24+
c =>
25+
c.Indexes.CreateOrUpdateAsync(
26+
new CreateIndexModel<DeletedFile>(Builders<DeletedFile>.IndexKeys.Ascending(p => p.DeletedAt))
27+
),
28+
]
2329
);
2430
configurator.AddRepository<Corpus>(
2531
"corpora.corpus",
26-
init: async c =>
27-
{
28-
await c.Indexes.CreateOrUpdateAsync(
29-
new CreateIndexModel<Corpus>(Builders<Corpus>.IndexKeys.Ascending(p => p.Owner))
30-
);
32+
init:
33+
[
34+
c =>
35+
c.Indexes.CreateOrUpdateAsync(
36+
new CreateIndexModel<Corpus>(Builders<Corpus>.IndexKeys.Ascending(p => p.Owner))
37+
),
3138
// migrate by adding Name field
32-
await c.UpdateManyAsync(
33-
Builders<Corpus>.Filter.Exists(b => b.Name, false),
34-
Builders<Corpus>.Update.Set(b => b.Name, null)
35-
);
36-
}
39+
c =>
40+
c.UpdateManyAsync(
41+
Builders<Corpus>.Filter.Exists(b => b.Name, false),
42+
Builders<Corpus>.Update.Set(b => b.Name, null)
43+
),
44+
]
3745
);
3846
return configurator;
3947
}

0 commit comments

Comments
 (0)