From 55aff77ff737e9fa6aa74ee5d198169cede10548 Mon Sep 17 00:00:00 2001 From: deevil Date: Sat, 16 Sep 2023 17:03:11 +0300 Subject: [PATCH 1/2] Add store mode Add store mode Add store mode Add store mode: --- .../ClickHouse/ClickHouseStorage.cs | 29 ++++++++++++++----- .../ExportersManager.cs | 9 +++--- .../appsettings.json | 3 +- README.md | 14 +++++++-- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs b/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs index 007e843..cf82a13 100644 --- a/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs +++ b/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs @@ -18,11 +18,16 @@ public class ClickHouseStorage : IEventLogStorage private ClickHouseConnection _connection; private string _connectionString; private string _databaseName; + private string _tableName; + private readonly int _storeMode; + private readonly string _targetName; - public ClickHouseStorage(string connectionsString, ILogger logger = null) + public ClickHouseStorage(string connectionsString, ILogger logger = null, string targetName = "", int storeMode = 1) { _logger = logger; _connectionString = connectionsString; + _storeMode = storeMode; + _targetName = targetName; Init(); } @@ -31,7 +36,7 @@ public ClickHouseStorage(ILogger logger, IConfiguration confi { _logger = logger; _connectionString = configuration.GetValue("ClickHouse:ConnectionString", ""); - + Init(); } @@ -40,7 +45,7 @@ public async Task ReadEventLogPositionAsync(CancellationToken await CreateConnectionAsync(cancellationToken); var commandText = - $"SELECT TOP 1 FileName, EndPosition, LgfEndPosition, Id FROM {TableName} ORDER BY DateTime DESC, EndPosition DESC"; + $"SELECT TOP 1 FileName, EndPosition, LgfEndPosition, Id FROM {_tableName} ORDER BY DateTime DESC, EndPosition DESC"; await using var cmd = _connection.CreateCommand(); cmd.CommandText = commandText; @@ -60,7 +65,7 @@ public async Task WriteEventLogDataAsync(List entities, using var copy = new ClickHouseBulkCopy(_connection) { - DestinationTableName = TableName, + DestinationTableName = _tableName, BatchSize = entities.Count }; @@ -98,11 +103,11 @@ public async Task WriteEventLogDataAsync(List entities, } catch (Exception ex) { - _logger?.LogError(ex, $"Failed to write data to {_databaseName}"); + _logger?.LogError(ex, $"Failed to write data to {_databaseName}.{_tableName}"); throw; } - _logger?.LogDebug($"{entities.Count} items were being written to {_databaseName}"); + _logger?.LogDebug($"{entities.Count} items were being written to {_databaseName}.{_tableName}"); } public void Dispose() @@ -122,6 +127,16 @@ private void Init() throw new Exception("Database name is not specified"); else _databaseName = FixDatabaseName(_databaseName); + + if (_storeMode == 2) { + // in tables + _tableName = _targetName; + + } else { + // in databases + _tableName = TableName; + _databaseName = _targetName; + } } private static string FixDatabaseName(string name) @@ -149,7 +164,7 @@ private async Task CreateEventLogItemsDatabaseAsync(CancellationToken cancellati await _connection.ChangeDatabaseAsync(_databaseName, cancellationToken); var commandText = - $@"CREATE TABLE IF NOT EXISTS {TableName} + $@"CREATE TABLE IF NOT EXISTS {_tableName} ( FileName LowCardinality(String), EndPosition Int64 Codec(DoubleDelta, LZ4), diff --git a/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs b/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs index 7e8fde4..31d8f44 100644 --- a/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs +++ b/OneSTools.EventLog.Exporter.Manager/ExportersManager.cs @@ -26,6 +26,7 @@ public class ExportersManager : BackgroundService private readonly bool _loadArchive; private readonly ILogger _logger; private readonly int _maximumRetries; + private readonly int _storeMode; private readonly TimeSpan _maxRetryTimeout; @@ -72,6 +73,7 @@ public ExportersManager(ILogger logger, IServiceProvider servi case StorageType.ClickHouse: { _connectionString = configuration.GetValue("ClickHouse:ConnectionString", ""); + _storeMode = configuration.GetValue("ClickHouse:StoreMode", 1); if (_connectionString == string.Empty) throw new Exception("Connection string is not specified"); break; @@ -227,11 +229,10 @@ private IEventLogStorage GetStorage(string dataBaseName) { case StorageType.ClickHouse: { - var logger = - (ILogger)_serviceProvider.GetService(typeof(ILogger)); - var connectionString = $"{_connectionString}Database={dataBaseName};"; + var logger = (ILogger)_serviceProvider.GetService(typeof(ILogger)); + //var connectionString = $"{_connectionString}Database={dataBaseName};"; - return new ClickHouseStorage(connectionString, logger); + return new ClickHouseStorage(_connectionString, logger, dataBaseName, _storeMode); } case StorageType.ElasticSearch: { diff --git a/OneSTools.EventLog.Exporter.Manager/appsettings.json b/OneSTools.EventLog.Exporter.Manager/appsettings.json index 852e927..3d580aa 100644 --- a/OneSTools.EventLog.Exporter.Manager/appsettings.json +++ b/OneSTools.EventLog.Exporter.Manager/appsettings.json @@ -30,7 +30,8 @@ "SkipEventsBeforeDate": "2022-04-01T00:00:00" }, "ClickHouse": { - "ConnectionString": "Host=172.19.149.61;Port=8123;Username=default;password=;" + "ConnectionString": "Host=172.19.149.61;Port=8123;Username=default;password=;", + "StoreMode": 2 }, "ElasticSearch": { "Nodes": [ diff --git a/README.md b/README.md index c6e7494..b41ac7e 100644 --- a/README.md +++ b/README.md @@ -88,10 +88,17 @@ **ClickHouse:** ```json -"ConnectionStrings": { - "Default": "Host=localhost;Port=8123;Username=default;password=;Database=database_name;" +"ClickHouse": { + "ConnectionString": "Host=localhost;Port=8123;Username=default;password=;Database=database_name;", + "StoreMode": 1 } ``` +где:
+1. *ConnectionString* - строка подключения к ClickHouse +2. *StoreMode* - как хранить различные журналы регистраций:
+ *1* - Каждый журнал регистрации - отдельная **база данных**
+ *2* - Каждый журнал регитрации - отдельная **таблица** + **ElasticSearch:** ```json "ElasticSearch": { @@ -169,7 +176,8 @@ "LoadArchive": false }, "ClickHouse": { - "ConnectionString": "Host=192.168.0.93;Port=8123;Database=upp_main_el;Username=default;password=;" + "ConnectionString": "Host=192.168.0.93;Port=8123;Database=upp_main_el;Username=default;password=;", + "StoreMode": 1 }, "ElasticSearch": { "Nodes": [ From fb0c77ed01e1bd22193e954e9608b63ecc8dfa8d Mon Sep 17 00:00:00 2001 From: deevil Date: Tue, 17 Oct 2023 14:39:30 +0300 Subject: [PATCH 2/2] Fix CH StoreMode for Exporter --- .../ClickHouse/ClickHouseStorage.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs b/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs index cf82a13..e7a8b6c 100644 --- a/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs +++ b/OneSTools.EventLog.Exporter.Core/ClickHouse/ClickHouseStorage.cs @@ -24,6 +24,7 @@ public class ClickHouseStorage : IEventLogStorage public ClickHouseStorage(string connectionsString, ILogger logger = null, string targetName = "", int storeMode = 1) { + // Constructor - EventLogExportersManager _logger = logger; _connectionString = connectionsString; _storeMode = storeMode; @@ -34,9 +35,13 @@ public ClickHouseStorage(string connectionsString, ILogger lo public ClickHouseStorage(ILogger logger, IConfiguration configuration) { + // Constructor - EventLogExporter _logger = logger; _connectionString = configuration.GetValue("ClickHouse:ConnectionString", ""); - + // Use DB from connection string + default table name + _storeMode = 2; + _targetName = TableName; + Init(); }