mirror of
https://github.com/hanzoai/jube-fork.git
synced 2026-10-30 18:33:33 +00:00
richard-churchman
7eedc7ee25
* Implemented a wrapper library for Redis and Postgres clients using Polly retry to enhance durability during failover. Removed all direct client references across projects, replacing them with the new resilient libraries. Bulk changes were largely confined to the database context constructor, given the application's broad use of the Repository pattern and LINQ2DB. * Implemented a WAL log in Redis that stores JSON payloads pending commit to the Archive table as part of an asynchronous process. The WAL entry is removed once the commit completes. During graceful drain on SIGTERM, this provides an observability and recovery mechanism for transactions lost due to host failure or fatal termination. * Implemented a read-only reporting connection for all case management query functions and other backend functions with significant read overhead or security implications, enabling targeting of read replicas under read-only database users. Dynamic query functions have been handled carefully to ensure parameterised variables are always used. * Implemented a `MigrationConnectionString` environment variable to support a dedicated migration user, allowing the application and reporting to run under minimal permissions while still supporting the online upgrade process used in Jube Herd (Docker Swarm). * Added evaluation counters to Gateway and Activation Rule Counters, and ensured they are displayed in the Gateway Rules, Activation Rules, and Activation Rules pages — an important missing observability feature for gradual rule rollout assessment prior to taking decline or case creation steps. Also introduced model total response time counters throughout the invocation pipeline and updated the administration page accordingly. Made all response times throughout the system record in microseconds for consistency. * Updated `CookieOptions` to include the `Secure` flag having been flagged on CodeQL. * Fixed major bugs in fan-out queries across all Case Management entities caused by the recently released permissions functionality, where entities were duplicated when more than one user was present. * Fixed a long-standing issue with performance counter increments to use `Interlocked`, or custom locks in the case of date-based counters. * Fixed a bug in preservation import introduced during the `CacheIndexId` functionality, which failed due to a nested transaction. The import now bypasses `CacheIndexId` rekeying and reads it directly from the `.jemp` file, which was itself the correct original behaviour. * Fixed Payload Responses not being included in the response payload when response payload was selected in Request XPath. * Fixed the Report Table switch in Request XPath having no effect and not writing to the `ArchiveKeys` table. * Fixed the inability to edit Case Workflows in the user interface because of a minor JS typo in the key name. * Refactored notification dispatch to use async methods for all database calls. * Refactored the Postgres Query class to make better use of `using`/dispose patterns, moved connection creation to the constructor, and ensured consistent disposal. Extended the use of `using` dispose throughout all LINQ2DB queries outside the Repository pattern. * Some refactoring to use byte arrays rather than `MemoryStream` in serialisation of JSON Archive and response payloads. Eliminated duplication in Archive payload creation for case creation, given the payload now also exists in the WAL, and is created during invocation (not asynchronously as was the case). * Updated the composite index for `AddArchiveKeyTableIndex` in Baseline to use covering indexes ONLY, avoiding Postgres page lookups (noting this table exists solely as an access path to the Archive table via predication and join on `EntityAnalysisModelInstanceEntryGuid`). IMPORTANT NOTE: No migration provided for existing data. Any such index will need to be rebuilt manually to obtain this benefit. * Updated the completions controller and query to order fields by name ascending, with new repository methods to apply ordering server-side. * Updated the date format for Created Dates in the user interface to a more verbose and readable format, consistent with the rest of the system. * Removed legacy billing aggregation counter for response elevation, which is no longer a relevant concept. Renamed remaining billing-related counter fields for clarity. * Updated Dockerfile for the new resilience projects. * Fixed typo in TTL Counter Resolution page where hours was duplicated instead of minutes. * Implemented RedisBackplane environment variable to support SignalR with Redis in clustered environments. * Fixed loss of password, password expiry, and password created date during User Administration updates; added empty password check in authentication controller to raise an exception treated as user not found; fixed inability to change password on expiry; fixed password expiry date evaluation bug; moved increment password failure to capture all code paths. * Updated ExhaustiveSearchInstanceTrialInstanceVariable repository to use logical deletes instead of physical deletes, which are unsupported by the database user; updated MockArchive loading to occur only once since the data is static. * Updated default Log4NetLogLevel environment variable to WARN and updated documentation accordingly. * Fixed missing reference for Archive Keys during payload creation, as while they were being populated, the reference to it was missing, so they did not get stored.
635 lines
32 KiB
C#
635 lines
32 KiB
C#
/* Copyright (C) 2022-present Jube Holdings Limited.
|
|
*
|
|
* This file is part of Jube™ software.
|
|
*
|
|
* Jube™ is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License
|
|
* as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
|
* Jube™ is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
|
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
|
|
|
|
* You should have received a copy of the GNU Affero General Public License along with Jube™. If not,
|
|
* see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
namespace Jube.Engine.BackgroundTasks.TaskStarters
|
|
{
|
|
using System;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Context;
|
|
using Data.Context;
|
|
using Data.Poco;
|
|
using Data.Repository;
|
|
using Microsoft.VisualBasic;
|
|
using EntityAnalysisModel=EntityAnalysisModelManager.EntityAnalysisModel.EntityAnalysisModel;
|
|
|
|
public class ManageCountersStarter(Context context)
|
|
{
|
|
public async Task StartAsync()
|
|
{
|
|
try
|
|
{
|
|
while (!context.Services.TaskCoordinator.CancellationToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
foreach (var (key, value) in
|
|
from modelEntityKvp in context.Tasks.EntityAnalysisModelManager.Context.EntityAnalysisModels.ActiveEntityAnalysisModels
|
|
where modelEntityKvp.Value.Counters.LastCountersChecked.AddMilliseconds(1000) < DateTime.Now
|
|
select modelEntityKvp)
|
|
{
|
|
ClearResponseElevationCounters(value, context.Services.TaskCoordinator.CancellationToken);
|
|
ClearFrequencyCounters(value, context.Services.TaskCoordinator.CancellationToken);
|
|
ClearActivationWatcherCounters(value, context.Services.TaskCoordinator.CancellationToken);
|
|
await UpdateQueueBalancesInDatabaseAtModelLevelAsync(value, context.Services.TaskCoordinator.CancellationToken).ConfigureAwait(false);
|
|
await UpdateCountersInDatabaseAsync(value, key, context.Services.TaskCoordinator.CancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
await UpdateHttpCountersInDatabaseAsync(context.Services.TaskCoordinator.CancellationToken).ConfigureAwait(false);
|
|
|
|
await UpdateQueueBalancesInDatabaseAsync(context.Services.TaskCoordinator.CancellationToken).ConfigureAwait(false);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: Counters written.");
|
|
}
|
|
|
|
await Task.Delay(1000, context.Services.TaskCoordinator.CancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error($"Counter Management: An error in counter management has been observed as {ex}.");
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException ex)
|
|
{
|
|
context.Services.Log.Info($"Graceful Cancellation ManageCountersAsync: has produced an error {ex}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
context.Services.Log.Error($"ManageCountersAsync: has produced an error {ex}");
|
|
}
|
|
}
|
|
|
|
private void ClearResponseElevationCounters(EntityAnalysisModel value, CancellationToken token = default)
|
|
{
|
|
try
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Clearing Billing Response Elevation Balance Entries queue.");
|
|
}
|
|
|
|
var cancelled = token.IsCancellationRequested;
|
|
|
|
while (value.ConcurrentQueues.ResponseElevationEntries.TryPeek(out var entry))
|
|
{
|
|
if (cancelled)
|
|
{
|
|
value.ConcurrentQueues.ResponseElevationEntries.Clear();
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Cancellation requested. Drained Billing Response Elevation Balance Entries queue.");
|
|
}
|
|
break;
|
|
}
|
|
|
|
var expiry = DateAndTime.DateAdd(
|
|
value.Counters.MaxResponseElevationInterval.ToString(),
|
|
value.Counters.MaxResponseElevationValue,
|
|
entry.CreatedDate);
|
|
|
|
if (DateTime.Now <= expiry)
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: No expired entries to remove from Billing Response Elevation Balance Entries queue.");
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (!value.ConcurrentQueues.ResponseElevationEntries.TryDequeue(out entry))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug($"Counter Management: Removed entry with value {entry.Value}.");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error($"ClearResponseElevationCounters: has produced an error {ex}");
|
|
}
|
|
}
|
|
|
|
private void ClearFrequencyCounters(EntityAnalysisModel value, CancellationToken token = default)
|
|
{
|
|
try
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Clearing Billing Response Elevation frequency Journal queue.");
|
|
}
|
|
|
|
var cancelled = token.IsCancellationRequested;
|
|
|
|
while (value.ConcurrentQueues.BillingResponseElevationJournal.TryPeek(out var responseElevationDate))
|
|
{
|
|
if (cancelled)
|
|
{
|
|
value.ConcurrentQueues.BillingResponseElevationJournal.Clear();
|
|
Interlocked.Exchange(ref value.Counters.ResponseElevationCount, 0);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Cancellation requested. Drained Billing Response Elevation queue.");
|
|
}
|
|
break;
|
|
}
|
|
|
|
var expiry = DateAndTime.DateAdd(
|
|
value.Counters.MaxResponseElevationInterval.ToString(),
|
|
value.Counters.MaxResponseElevationValue,
|
|
responseElevationDate);
|
|
|
|
if (DateTime.Now >= expiry)
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: No expired counters to remove from Billing Response Elevation queue.");
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (!value.ConcurrentQueues.BillingResponseElevationJournal.TryDequeue(out responseElevationDate))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
Interlocked.Decrement(ref value.Counters.ResponseElevationCount);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Removed entry with date {responseElevationDate}. Decremented Response Elevation Count to {value.Counters.ResponseElevationCount}.");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error($"ClearFrequencyCounters: has produced an error {ex}");
|
|
}
|
|
}
|
|
|
|
private void ClearActivationWatcherCounters(EntityAnalysisModel value, CancellationToken token = default)
|
|
{
|
|
try
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Clearing Activation Watcher Count Journal queue.");
|
|
}
|
|
|
|
var cancelled = token.IsCancellationRequested;
|
|
|
|
while (value.ConcurrentQueues.ActivationWatcherCountJournal.TryPeek(out var activationWatcherDate))
|
|
{
|
|
if (cancelled)
|
|
{
|
|
value.ConcurrentQueues.ActivationWatcherCountJournal.Clear();
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Cancellation requested. Drained Activation Watcher queue.");
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
var expiry = DateAndTime.DateAdd(value.Counters.MaxActivationWatcherInterval.ToString(),
|
|
value.Counters.MaxActivationWatcherValue, activationWatcherDate);
|
|
|
|
if (DateTime.Now <= expiry)
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: No expired counters to remove from Activation Watcher queue.");
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (!value.ConcurrentQueues.ActivationWatcherCountJournal.TryDequeue(out activationWatcherDate))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
Interlocked.Decrement(ref value.Counters.ActivationWatcherCount);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Removed entry with date {activationWatcherDate}. Decremented Activation Watcher Count to {value.Counters.ActivationWatcherCount}.");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error($"ClearActivationWatcherCounters: has produced an error {ex}");
|
|
}
|
|
}
|
|
|
|
private async Task UpdateQueueBalancesInDatabaseAtModelLevelAsync(EntityAnalysisModel value, CancellationToken token = default)
|
|
{
|
|
try
|
|
{
|
|
if (value.Counters.LastCountersWritten.AddSeconds(60) < DateTime.Now)
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Starting to store queue balances in the database.");
|
|
}
|
|
|
|
var dbContext =
|
|
DataConnectionDbContext.GetResilientDbContextDataConnection(
|
|
context.Services.DynamicEnvironment.AppSettings("ConnectionString"), context.Services.Log);
|
|
try
|
|
{
|
|
var repository = new EntityAnalysisModelAsynchronousQueueBalanceRepository(dbContext);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: Has opened a database connection to invoke Insert_Entity_Analysis_Models_Asynchronous_Queue_Balances.");
|
|
}
|
|
|
|
var insert = new EntityAnalysisModelAsynchronousQueueBalance
|
|
{
|
|
Archive = value.ConcurrentQueues.PersistToDatabaseAsync.Count,
|
|
EntityAnalysisModelGuid = value.Instance.Guid,
|
|
ActivationWatcher = value.ConcurrentQueues.PersistToDatabaseAsync.Count,
|
|
CreatedDate = DateTime.Now,
|
|
Instance = Dns.GetHostName()
|
|
};
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Has built the command object to invoke Insert_Entity_Analysis_Models_Asynchronous_Queue_Balances with Entity Analysis Model ID {value.Instance.Id}, Activation Cases 0, Activation Watcher {value.ConcurrentQueues.ResponseElevationEntries.Count}, Billing Response Elevation {value.ConcurrentQueues.ActivationWatcherCountJournal.Count},Billing_Response_Elevation_Balance {value.ConcurrentQueues.BillingResponseElevationJournal.Count}, Billing_Response_Elevation_Balance {value.ConcurrentQueues.BillingResponseElevationJournal.Count}, Node {context.Services.DynamicEnvironment.AppSettings("Node")}.");
|
|
}
|
|
|
|
await repository.InsertAsync(insert, token).ConfigureAwait(false);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: Has opened a database connection to invoke Insert_Entity_Analysis_Models_Asynchronous_Queue_Balances.");
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error(
|
|
$"Counter Management: There was an error invoking Insert_Entity_Analysis_Models_Asynchronous_Queue_Balances as {ex}.");
|
|
}
|
|
finally
|
|
{
|
|
await dbContext.CloseAsync(token).ConfigureAwait(false);
|
|
await dbContext.DisposeAsync(token).ConfigureAwait(false);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: Closed the database connection for invoking Insert_Entity_Analysis_Models_Asynchronous_Queue_Balances.");
|
|
}
|
|
}
|
|
|
|
value.Counters.LastCountersWritten = DateTime.Now;
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Updated last queue balances written {value.Counters.LastCountersWritten}.");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: Has not stored queue balances as the storage period has not lapsed, every 60 seconds.");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error($"UpdateQueueBalancesInDatabaseAtModelLevelAsync: has produced an error {ex}");
|
|
}
|
|
}
|
|
|
|
private async Task UpdateCountersInDatabaseAsync(EntityAnalysisModel value, int key, CancellationToken token = default)
|
|
{
|
|
try
|
|
{
|
|
if (value.Counters.LastModelInvokeCountersWritten.AddSeconds(60) < DateTime.Now)
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Starting to store model counters in the database.");
|
|
}
|
|
|
|
var dbContext = DataConnectionDbContext.GetResilientDbContextDataConnection(
|
|
context.Services.DynamicEnvironment.AppSettings("ConnectionString"), context.Services.Log);
|
|
try
|
|
{
|
|
var repository = new EntityAnalysisModelProcessingCounterRepository(dbContext);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Model {key} Has opened a database connection to invoke Insert_Entity_Analysis_Models_Processing_Counters.");
|
|
}
|
|
|
|
var model = new EntityAnalysisModelProcessingCounter
|
|
{
|
|
ModelInvoke = value.Counters.ModelInvokeCounter,
|
|
GatewayMatch = value.Counters.ModelInvokeGatewayCounter,
|
|
ResponseElevation = value.Counters.ModelResponseElevationCounter,
|
|
ResponseElevationValueLimit = value.Counters.ResponseElevationValueLimitCounter,
|
|
ResponseElevationLimit = value.Counters.ResponseElevationFrequencyLimitCounter,
|
|
ModelTotalResponseTime = value.Counters.ModelTotalResponseTime,
|
|
ActivationWatcher = value.Counters.ActivationWatcherCount,
|
|
EntityAnalysisModelGuid = value.Instance.Guid,
|
|
CreatedDate = DateTime.Now,
|
|
Instance = Dns.GetHostName()
|
|
};
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug($"Counter Management: Model {key} Has built command for Insert_Entity_Analysis_Models_Processing_Counters with Model_Invoke_Counter {value.Counters.ModelInvokeCounter},Gateway_Match_Counter {value.Counters.ModelInvokeGatewayCounter},Response_Elevation_Counter {value.Counters.ModelResponseElevationCounter},Response_Elevation_Value_Limit_Counter {value.Counters.ResponseElevationValueLimitCounter},Response_Elevation_Frequency_Limit_Counter {value.Counters.ResponseElevationFrequencyLimitCounter},{value.Instance.Id}.");
|
|
}
|
|
|
|
await repository.InsertAsync(model, token).ConfigureAwait(false);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Model {key} Has opened a database connection to invoke Insert_Entity_Analysis_Models_Processing_Counters.");
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error(
|
|
$"Counter Management: Model {key} There was an error invoking Insert_Entity_Analysis_Models_Processing_Counters as {ex}.");
|
|
}
|
|
finally
|
|
{
|
|
await dbContext.CloseAsync(token).ConfigureAwait(false);
|
|
await dbContext.DisposeAsync(token).ConfigureAwait(false);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Model {key} closed the database connection for invoking Insert_Entity_Analysis_Models_Processing_Counters.");
|
|
}
|
|
}
|
|
|
|
Interlocked.Exchange(ref value.Counters.ResponseElevationCount, 0);
|
|
Interlocked.Exchange(ref value.Counters.ModelInvokeCounter, 0);
|
|
Interlocked.Exchange(ref value.Counters.ModelInvokeGatewayCounter, 0);
|
|
Interlocked.Exchange(ref value.Counters.ModelResponseElevationCounter, 0);
|
|
Interlocked.Exchange(ref value.Counters.ModelTotalResponseTime, 0);
|
|
Interlocked.Exchange(ref value.Counters.ResponseElevationValueLimitCounter, 0);
|
|
Interlocked.Exchange(ref value.Counters.ResponseElevationFrequencyLimitCounter, 0);
|
|
Interlocked.Exchange(ref value.Counters.ActivationWatcherCount, 0);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Model {key} has reset all model counters.");
|
|
}
|
|
|
|
value.Counters.LastModelInvokeCountersWritten = DateTime.Now;
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Model {key} updated last counters written {value.Counters.LastModelInvokeCountersWritten}.");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Model {key} has not stored counters as the storage period has not lapsed, every 60 seconds.");
|
|
}
|
|
}
|
|
|
|
value.Counters.LastCountersChecked = DateTime.Now;
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: Model {key} updated last counters checked {value.Counters.LastCountersChecked}.");
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error($"UpdateCountersInDatabaseAsync: has produced an error {ex}");
|
|
}
|
|
}
|
|
|
|
private async Task UpdateHttpCountersInDatabaseAsync(CancellationToken token = default)
|
|
{
|
|
try
|
|
{
|
|
if (context.Counters.LastHttpCountersWritten.AddSeconds(60) < DateTime.Now)
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Starting to store HTTP counters in the database.");
|
|
}
|
|
|
|
var dbContext =
|
|
DataConnectionDbContext.GetResilientDbContextDataConnection(
|
|
context.Services.DynamicEnvironment.AppSettings("ConnectionString"), context.Services.Log);
|
|
try
|
|
{
|
|
var repository = new HttpProcessingCounterRepository(dbContext);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: Has opened a database connection to invoke Insert_HTTP_Processing_Counters.");
|
|
}
|
|
|
|
var model = new HttpProcessingCounter
|
|
{
|
|
Instance = Dns.GetHostName(),
|
|
All = context.Counters.HttpCounterAllRequests,
|
|
Model = context.Counters.HttpCounterModel,
|
|
AsynchronousModel = context.Counters.HttpCounterModelAsync,
|
|
Error = context.Counters.HttpCounterAllError,
|
|
Tag = context.Counters.HttpCounterTag,
|
|
Sanction = context.Counters.HttpCounterSanction,
|
|
Callback = context.Counters.HttpCounterCallback,
|
|
Exhaustive = context.Counters.HttpCounterExhaustive,
|
|
CreatedDate = DateTime.Now
|
|
};
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: has built command for Insert_HTTP_Processing_Counters with All_Activity " +
|
|
$"{context.Counters.HttpCounterAllRequests},Models {context.Counters.HttpCounterModel}," +
|
|
$"Async Models {context.Counters.HttpCounterModelAsync},Tag {context.Counters.HttpCounterTag},Errors {context.Counters.HttpCounterAllError}," +
|
|
$"Exhaustive {context.Counters.HttpCounterExhaustive} and Sanction {context.Counters.HttpCounterSanction}.");
|
|
}
|
|
|
|
await repository.InsertAsync(model, token).ConfigureAwait(false);
|
|
|
|
context.Counters.LastHttpCountersWritten = DateTime.Now;
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: has updated HTTP processing counters.");
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error(
|
|
$"Counter Management: error created updating HTTP processing counters as {ex}.");
|
|
}
|
|
finally
|
|
{
|
|
await dbContext.CloseAsync(token).ConfigureAwait(false);
|
|
await dbContext.DisposeAsync(token).ConfigureAwait(false);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: has updated HTTP processing counters and closed the database connection.");
|
|
}
|
|
}
|
|
|
|
Interlocked.Exchange(ref context.Counters.HttpCounterAllRequests, 0);
|
|
Interlocked.Exchange(ref context.Counters.HttpCounterModel, 0);
|
|
Interlocked.Exchange(ref context.Counters.HttpCounterModelAsync, 0);
|
|
Interlocked.Exchange(ref context.Counters.HttpCounterAllError, 0);
|
|
Interlocked.Exchange(ref context.Counters.HttpCounterTag, 0);
|
|
Interlocked.Exchange(ref context.Counters.HttpCounterSanction, 0);
|
|
Interlocked.Exchange(ref context.Counters.HttpCounterCallback, 0);
|
|
Interlocked.Exchange(ref context.Counters.HttpCounterExhaustive, 0);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: has updated HTTP processing counters reset.");
|
|
}
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: has updated HTTP processing counters last written date {context.Counters.LastHttpCountersWritten}.");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: Model has not stored HTTP counters as the storage period has not lapsed, every 60 seconds.");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error($"UpdateHttpCountersInDatabaseAsync: has produced an error {ex}");
|
|
}
|
|
}
|
|
|
|
private async Task UpdateQueueBalancesInDatabaseAsync(CancellationToken token = default)
|
|
{
|
|
try
|
|
{
|
|
if (context.Counters.LastBalanceCountersWritten.AddSeconds(60) >= DateTime.Now)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug("Counter Management: Starting to store in memory asynchronous queues in database.");
|
|
}
|
|
|
|
var dbContext = DataConnectionDbContext.GetResilientDbContextDataConnection(
|
|
context.Services.DynamicEnvironment.AppSettings("ConnectionString"), context.Services.Log);
|
|
try
|
|
{
|
|
var repository = new EntityAnalysisAsynchronousQueueBalanceRepository(dbContext);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: Has opened a database connection to invoke insertion of queue balance.");
|
|
}
|
|
|
|
var model = new EntityAnalysisAsynchronousQueueBalance
|
|
{
|
|
AsynchronousInvoke = context.ConcurrentQueues.PendingEntityInvoke.Count,
|
|
AsynchronousCallback = context.Services.CacheService.CacheCallbackPublishSubscribe.Callbacks.Count,
|
|
CaseCreation = context.ConcurrentQueues.PendingCases.Count,
|
|
Tagging = context.ConcurrentQueues.PendingTagging.Count,
|
|
Notification = context.ConcurrentQueues.PendingNotifications.Count,
|
|
CreatedDate = DateTime.Now,
|
|
Instance = Dns.GetHostName()
|
|
};
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
$"Counter Management: has built command to invoke insert of queue balance as Tagging {context.ConcurrentQueues.PendingTagging.Count} and Node {context.Services.DynamicEnvironment.AppSettings("Node")}. Has reset expired counters.");
|
|
}
|
|
|
|
await repository.InsertAsync(model, token).ConfigureAwait(false);
|
|
|
|
context.Counters.LastBalanceCountersWritten = DateTime.Now;
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: has updated HTTP processing counters.");
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error(
|
|
$"Counter Management: Invocation of queue balance insertion has an error as {ex}.");
|
|
}
|
|
finally
|
|
{
|
|
await dbContext.CloseAsync(token).ConfigureAwait(false);
|
|
await dbContext.DisposeAsync(token).ConfigureAwait(false);
|
|
|
|
if (context.Services.Log.IsDebugEnabled)
|
|
{
|
|
context.Services.Log.Debug(
|
|
"Counter Management: invocation of insert of HTTP processing counters has finished and the connection is closed.");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
context.Services.Log.Error($"UpdateQueueBalancesInDatabaseAsync: has produced an error {ex}");
|
|
}
|
|
}
|
|
}
|
|
}
|