diff --git a/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs b/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs index 007e843..e7a8b6c 100644 --- a/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs +++ b/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs @@ -18,19 +18,29 @@ public class ClickHouseStorage : IEventLogStorage private ClickHouseConnection _connection; private string _connectionString; private string _databaseName; + private string _tableName; + private readonly int _storeMode; + private readonly string _targetName; - public ClickHouseStorage(string connectionsString, ILogger logger = null) + public ClickHouseStorage(string connectionsString, ILogger logger = null, string targetName = "", int storeMode = 1) { + // Constructor - EventLogExportersManager _logger = logger; _connectionString = connectionsString; + _storeMode = storeMode; + _targetName = targetName; Init(); } public ClickHouseStorage(ILogger logger, IConfiguration configuration) { + // Constructor - EventLogExporter _logger = logger; _connectionString = configuration.GetValue("ClickHouse:ConnectionString", ""); + // Use DB from connection string + default table name + _storeMode = 2; + _targetName = TableName; Init(); } @@ -40,7 +50,7 @@ public async Task ReadEventLogPositionAsync(CancellationToken await CreateConnectionAsync(cancellationToken); var commandText = - $"SELECT TOP 1 FileName, EndPosition, LgfEndPosition, Id FROM {TableName} ORDER BY DateTime DESC, EndPosition DESC"; + $"SELECT TOP 1 FileName, EndPosition, LgfEndPosition, Id FROM {_tableName} ORDER BY DateTime DESC, EndPosition DESC"; await using var cmd = _connection.CreateCommand(); cmd.CommandText = commandText; @@ -60,7 +70,7 @@ public async Task WriteEventLogDataAsync(List entities, using var copy = new ClickHouseBulkCopy(_connection) { - DestinationTableName = TableName, + DestinationTableName = _tableName, BatchSize = entities.Count }; @@ -98,11 +108,11 @@ public async Task WriteEventLogDataAsync(List entities, } catch (Exception ex) { - _logger?.LogError(ex, $"Failed to write data to {_databaseName}"); + _logger?.LogError(ex, $"Failed to write data to {_databaseName}.{_tableName}"); throw; } - _logger?.LogDebug($"{entities.Count} items were being written to {_databaseName}"); + _logger?.LogDebug($"{entities.Count} items were being written to {_databaseName}.{_tableName}"); } public void Dispose() @@ -122,6 +132,16 @@ private void Init() throw new Exception("Database name is not specified"); else _databaseName = FixDatabaseName(_databaseName); + + if (_storeMode == 2) { + // in tables + _tableName = _targetName; + + } else { + // in databases + _tableName = TableName; + _databaseName = _targetName; + } } private static string FixDatabaseName(string name) @@ -149,7 +169,7 @@ private async Task CreateEventLogItemsDatabaseAsync(CancellationToken cancellati await _connection.ChangeDatabaseAsync(_databaseName, cancellationToken); var commandText = - $@"CREATE TABLE IF NOT EXISTS {TableName} + $@"CREATE TABLE IF NOT EXISTS {_tableName} ( FileName LowCardinality(String), EndPosition Int64 Codec(DoubleDelta, LZ4), diff --git a/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs b/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs index 7e8fde4..31d8f44 100644 --- a/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs +++ b/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs @@ -26,6 +26,7 @@ public class ExportersManager : BackgroundService private readonly bool _loadArchive; private readonly ILogger _logger; private readonly int _maximumRetries; + private readonly int _storeMode; private readonly TimeSpan _maxRetryTimeout; @@ -72,6 +73,7 @@ public ExportersManager(ILogger logger, IServiceProvider servi case StorageType.ClickHouse: { _connectionString = configuration.GetValue("ClickHouse:ConnectionString", ""); + _storeMode = configuration.GetValue("ClickHouse:StoreMode", 1); if (_connectionString == string.Empty) throw new Exception("Connection string is not specified"); break; @@ -227,11 +229,10 @@ private IEventLogStorage GetStorage(string dataBaseName) { case StorageType.ClickHouse: { - var logger = - (ILogger)_serviceProvider.GetService(typeof(ILogger)); - var connectionString = $"{_connectionString}Database={dataBaseName};"; + var logger = (ILogger)_serviceProvider.GetService(typeof(ILogger)); + //var connectionString = $"{_connectionString}Database={dataBaseName};"; - return new ClickHouseStorage(connectionString, logger); + return new ClickHouseStorage(_connectionString, logger, dataBaseName, _storeMode); } case StorageType.ElasticSearch: { diff --git a/OneSTools.EventLog.Exporter.Manager/appsettings.json b/OneSTools.EventLog.Exporter.Manager/appsettings.json index 852e927..3d580aa 100644 --- a/OneSTools.EventLog.Exporter.Manager/appsettings.json +++ b/OneSTools.EventLog.Exporter.Manager/appsettings.json @@ -30,7 +30,8 @@ "SkipEventsBeforeDate": "2022-04-01T00:00:00" }, "ClickHouse": { - "ConnectionString": "Host=172.19.149.61;Port=8123;Username=default;password=;" + "ConnectionString": "Host=172.19.149.61;Port=8123;Username=default;password=;", + "StoreMode": 2 }, "ElasticSearch": { "Nodes": [ diff --git a/README.md b/README.md index c6e7494..b41ac7e 100644 --- a/README.md +++ b/README.md @@ -88,10 +88,17 @@ **ClickHouse:** ```json -"ConnectionStrings": { - "Default": "Host=localhost;Port=8123;Username=default;password=;Database=database_name;" +"ClickHouse": { + "ConnectionString": "Host=localhost;Port=8123;Username=default;password=;Database=database_name;", + "StoreMode": 1 } ``` +где:
+1. *ConnectionString* - строка подключения к ClickHouse +2. *StoreMode* - как хранить различные журналы регистраций:
+ *1* - Каждый журнал регистрации - отдельная **база данных**
+ *2* - Каждый журнал регитрации - отдельная **таблица** + **ElasticSearch:** ```json "ElasticSearch": { @@ -169,7 +176,8 @@ "LoadArchive": false }, "ClickHouse": { - "ConnectionString": "Host=192.168.0.93;Port=8123;Database=upp_main_el;Username=default;password=;" + "ConnectionString": "Host=192.168.0.93;Port=8123;Database=upp_main_el;Username=default;password=;", + "StoreMode": 1 }, "ElasticSearch": { "Nodes": [