| | | 1 | | using System.Security.Cryptography; |
| | | 2 | | using System.Text; |
| | | 3 | | using Microsoft.EntityFrameworkCore; |
| | | 4 | | using Microsoft.Extensions.Options; |
| | | 5 | | using ProjectTemplate.Infrastructure.Data.Entities; |
| | | 6 | | |
| | | 7 | | namespace ProjectTemplate.Infrastructure.Data.Auditing; |
| | | 8 | | |
| | 16 | 9 | | public sealed class ApplicationAuditReconciler( |
| | 16 | 10 | | ApplicationDbContext dbContext, |
| | 16 | 11 | | IApplicationMutationManifestVerifier manifestVerifier, |
| | 16 | 12 | | IOptions<ApplicationAuditReconciliationOptions> options, |
| | 16 | 13 | | ApplicationAuditReconciliationMetrics metrics, |
| | 16 | 14 | | TimeProvider timeProvider) |
| | | 15 | | : IApplicationAuditReconciler |
| | | 16 | | { |
| | 16 | 17 | | private readonly ApplicationDbContext _dbContext = |
| | 16 | 18 | | dbContext ?? throw new ArgumentNullException(nameof(dbContext)); |
| | 16 | 19 | | private readonly IApplicationMutationManifestVerifier _manifestVerifier = |
| | 16 | 20 | | manifestVerifier ?? throw new ArgumentNullException(nameof(manifestVerifier)); |
| | 16 | 21 | | private readonly ApplicationAuditReconciliationOptions _options = |
| | 16 | 22 | | options?.Value ?? throw new ArgumentNullException(nameof(options)); |
| | 16 | 23 | | private readonly ApplicationAuditReconciliationMetrics _metrics = |
| | 16 | 24 | | metrics ?? throw new ArgumentNullException(nameof(metrics)); |
| | 16 | 25 | | private readonly TimeProvider _timeProvider = |
| | 16 | 26 | | timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); |
| | | 27 | | |
| | | 28 | | public async Task<ApplicationAuditReconciliationSummary> ReconcileAsync( |
| | | 29 | | CancellationToken cancellationToken = default) |
| | | 30 | | { |
| | 18 | 31 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 18 | 32 | | if (!_options.Enabled) |
| | | 33 | | { |
| | 2 | 34 | | return ApplicationAuditReconciliationMetrics.DisabledSummary; |
| | | 35 | | } |
| | | 36 | | |
| | 16 | 37 | | DateTime now = UtcNow(); |
| | 16 | 38 | | List<string> batchIds = await _dbContext.AuditRecords |
| | 16 | 39 | | .AsNoTracking() |
| | 16 | 40 | | .Where(record => record.MutationBatchId != string.Empty) |
| | 16 | 41 | | .GroupBy(record => record.MutationBatchId) |
| | 16 | 42 | | .OrderByDescending(group => group.Max(record => record.ModifiedOnUtc)) |
| | 16 | 43 | | .Select(group => group.Key) |
| | 16 | 44 | | .Take(_options.MaximumBatchesPerRun) |
| | 16 | 45 | | .ToListAsync(cancellationToken) |
| | 16 | 46 | | .ConfigureAwait(false); |
| | | 47 | | |
| | 16 | 48 | | List<AuditRecord> auditRecords = await _dbContext.AuditRecords |
| | 16 | 49 | | .AsNoTracking() |
| | 16 | 50 | | .Where(record => batchIds.Contains(record.MutationBatchId) || record.MutationBatchId == string.Empty) |
| | 16 | 51 | | .ToListAsync(cancellationToken) |
| | 16 | 52 | | .ConfigureAwait(false); |
| | | 53 | | |
| | 16 | 54 | | List<ApplicationAuditCompletionOutboxEntry> completionEntries = await _dbContext |
| | 16 | 55 | | .ApplicationAuditCompletionOutboxEntries |
| | 16 | 56 | | .AsNoTracking() |
| | 16 | 57 | | .Where(entry => batchIds.Contains(entry.MutationBatchId) || |
| | 16 | 58 | | entry.Status != ApplicationAuditCompletionOutboxStatuses.Delivered) |
| | 16 | 59 | | .OrderByDescending(entry => entry.CreatedUtc) |
| | 16 | 60 | | .Take(_options.MaximumBatchesPerRun * 2) |
| | 16 | 61 | | .ToListAsync(cancellationToken) |
| | 16 | 62 | | .ConfigureAwait(false); |
| | | 63 | | |
| | 16 | 64 | | List<ApplicationAuditReconciliationCandidate> candidates = BuildCandidates( |
| | 16 | 65 | | auditRecords, |
| | 16 | 66 | | completionEntries, |
| | 16 | 67 | | now); |
| | | 68 | | |
| | 16 | 69 | | await PersistCandidatesAsync(candidates, batchIds, completionEntries, now, cancellationToken) |
| | 16 | 70 | | .ConfigureAwait(false); |
| | | 71 | | |
| | 16 | 72 | | ApplicationAuditReconciliationSummary summary = await GetSummaryCoreAsync(now, cancellationToken) |
| | 16 | 73 | | .ConfigureAwait(false); |
| | 16 | 74 | | _metrics.Update(summary); |
| | 16 | 75 | | return summary; |
| | 18 | 76 | | } |
| | | 77 | | |
| | | 78 | | public async Task<ApplicationAuditReconciliationSummary> GetSummaryAsync( |
| | | 79 | | CancellationToken cancellationToken = default) |
| | | 80 | | { |
| | 0 | 81 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 0 | 82 | | if (!_options.Enabled) |
| | | 83 | | { |
| | 0 | 84 | | return ApplicationAuditReconciliationMetrics.DisabledSummary; |
| | | 85 | | } |
| | | 86 | | |
| | 0 | 87 | | ApplicationAuditReconciliationSummary summary = await GetSummaryCoreAsync( |
| | 0 | 88 | | _metrics.LastRunUtc, |
| | 0 | 89 | | cancellationToken) |
| | 0 | 90 | | .ConfigureAwait(false); |
| | 0 | 91 | | _metrics.Update(summary); |
| | 0 | 92 | | return summary; |
| | 0 | 93 | | } |
| | | 94 | | |
| | | 95 | | public async Task<IReadOnlyList<ApplicationAuditReconciliationFindingItem>> QueryFindingsAsync( |
| | | 96 | | ApplicationAuditReconciliationQuery request, |
| | | 97 | | CancellationToken cancellationToken = default) |
| | | 98 | | { |
| | 0 | 99 | | ArgumentNullException.ThrowIfNull(request); |
| | 0 | 100 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 101 | | |
| | 0 | 102 | | IQueryable<ApplicationAuditReconciliationFinding> query = _dbContext |
| | 0 | 103 | | .ApplicationAuditReconciliationFindings |
| | 0 | 104 | | .AsNoTracking(); |
| | | 105 | | |
| | 0 | 106 | | if (!string.IsNullOrWhiteSpace(request.ReasonCode)) |
| | | 107 | | { |
| | 0 | 108 | | string reasonCode = request.ReasonCode.Trim(); |
| | 0 | 109 | | query = query.Where(finding => finding.ReasonCode == reasonCode); |
| | | 110 | | } |
| | | 111 | | |
| | 0 | 112 | | if (!string.IsNullOrWhiteSpace(request.Severity)) |
| | | 113 | | { |
| | 0 | 114 | | string severity = request.Severity.Trim(); |
| | 0 | 115 | | query = query.Where(finding => finding.Severity == severity); |
| | | 116 | | } |
| | | 117 | | |
| | 0 | 118 | | if (!string.IsNullOrWhiteSpace(request.MutationBatchId)) |
| | | 119 | | { |
| | 0 | 120 | | string batchId = request.MutationBatchId.Trim(); |
| | 0 | 121 | | query = query.Where(finding => finding.MutationBatchId == batchId); |
| | | 122 | | } |
| | | 123 | | |
| | 0 | 124 | | if (!string.IsNullOrWhiteSpace(request.RemediationStatus)) |
| | | 125 | | { |
| | 0 | 126 | | string status = request.RemediationStatus.Trim(); |
| | 0 | 127 | | query = query.Where(finding => finding.RemediationStatus == status); |
| | | 128 | | } |
| | | 129 | | |
| | 0 | 130 | | int maximumResults = Math.Clamp(request.MaximumResults, 1, 500); |
| | 0 | 131 | | return await query |
| | 0 | 132 | | .OrderByDescending(finding => finding.LastObservedUtc) |
| | 0 | 133 | | .Take(maximumResults) |
| | 0 | 134 | | .Select(finding => new ApplicationAuditReconciliationFindingItem( |
| | 0 | 135 | | finding.Id, |
| | 0 | 136 | | finding.SchemaVersion, |
| | 0 | 137 | | finding.FindingKey, |
| | 0 | 138 | | finding.ReasonCode, |
| | 0 | 139 | | finding.Severity, |
| | 0 | 140 | | finding.MutationBatchId, |
| | 0 | 141 | | finding.Destination, |
| | 0 | 142 | | finding.Guidance, |
| | 0 | 143 | | finding.RemediationStatus, |
| | 0 | 144 | | finding.FirstObservedUtc, |
| | 0 | 145 | | finding.LastObservedUtc, |
| | 0 | 146 | | finding.ResolvedUtc)) |
| | 0 | 147 | | .ToListAsync(cancellationToken) |
| | 0 | 148 | | .ConfigureAwait(false); |
| | 0 | 149 | | } |
| | | 150 | | |
| | | 151 | | public async Task<ApplicationAuditReconciliationRemediationItem> RecordRemediationAsync( |
| | | 152 | | Guid findingId, |
| | | 153 | | ApplicationAuditReconciliationRemediationRequest request, |
| | | 154 | | CancellationToken cancellationToken = default) |
| | | 155 | | { |
| | 2 | 156 | | ArgumentNullException.ThrowIfNull(request); |
| | 2 | 157 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 158 | | |
| | 2 | 159 | | ApplicationAuditReconciliationFinding finding = await _dbContext |
| | 2 | 160 | | .ApplicationAuditReconciliationFindings |
| | 2 | 161 | | .AsNoTracking() |
| | 2 | 162 | | .SingleOrDefaultAsync(item => item.Id == findingId, cancellationToken) |
| | 2 | 163 | | .ConfigureAwait(false) |
| | 2 | 164 | | ?? throw new KeyNotFoundException($"Audit reconciliation finding '{findingId}' was not found."); |
| | | 165 | | |
| | 2 | 166 | | string actionCode = NormalizeRequired(request.ActionCode, 64, nameof(request.ActionCode)); |
| | 2 | 167 | | string actorId = NormalizeRequired(request.ActorId, 256, nameof(request.ActorId)); |
| | 2 | 168 | | string? evidenceReference = NormalizeOptional(request.EvidenceReference, 256); |
| | 2 | 169 | | DateTime now = UtcNow(); |
| | 2 | 170 | | var remediationId = Guid.NewGuid(); |
| | 2 | 171 | | string concurrencyStamp = Guid.NewGuid().ToString("N"); |
| | | 172 | | |
| | 2 | 173 | | await _dbContext.Database.ExecuteSqlInterpolatedAsync($$""" |
| | 2 | 174 | | INSERT INTO [ApplicationAuditReconciliationRemediations] |
| | 2 | 175 | | ([Id], [FindingId], [MutationBatchId], [ActionCode], [ActorId], [EvidenceReference], [RecordedUtc], [Con |
| | 2 | 176 | | VALUES |
| | 2 | 177 | | ({{remediationId}}, {{findingId}}, {{finding.MutationBatchId}}, {{actionCode}}, {{actorId}}, {{evidenceR |
| | 2 | 178 | | """, cancellationToken).ConfigureAwait(false); |
| | | 179 | | |
| | 2 | 180 | | string remediationStatus = request.ResolveFinding |
| | 2 | 181 | | ? ApplicationAuditReconciliationRemediationStatuses.Resolved |
| | 2 | 182 | | : ApplicationAuditReconciliationRemediationStatuses.Acknowledged; |
| | 2 | 183 | | DateTime? resolvedUtc = request.ResolveFinding ? now : null; |
| | | 184 | | |
| | 2 | 185 | | await _dbContext.Database.ExecuteSqlInterpolatedAsync($$""" |
| | 2 | 186 | | UPDATE [ApplicationAuditReconciliationFindings] |
| | 2 | 187 | | SET [RemediationStatus] = {{remediationStatus}}, |
| | 2 | 188 | | [ResolvedUtc] = {{resolvedUtc}}, |
| | 2 | 189 | | [ConcurrencyStamp] = {{Guid.NewGuid().ToString("N")}} |
| | 2 | 190 | | WHERE [Id] = {{findingId}} |
| | 2 | 191 | | """, cancellationToken).ConfigureAwait(false); |
| | | 192 | | |
| | 2 | 193 | | return new( |
| | 2 | 194 | | remediationId, |
| | 2 | 195 | | findingId, |
| | 2 | 196 | | finding.MutationBatchId, |
| | 2 | 197 | | actionCode, |
| | 2 | 198 | | actorId, |
| | 2 | 199 | | evidenceReference, |
| | 2 | 200 | | now); |
| | 2 | 201 | | } |
| | | 202 | | |
| | | 203 | | private List<ApplicationAuditReconciliationCandidate> BuildCandidates( |
| | | 204 | | IReadOnlyCollection<AuditRecord> auditRecords, |
| | | 205 | | IReadOnlyCollection<ApplicationAuditCompletionOutboxEntry> completionEntries, |
| | | 206 | | DateTime now) |
| | | 207 | | { |
| | 16 | 208 | | var candidates = new Dictionary<string, ApplicationAuditReconciliationCandidate>(StringComparer.Ordinal); |
| | 16 | 209 | | ILookup<string, AuditRecord> auditBatches = auditRecords |
| | 16 | 210 | | .Where(record => !string.IsNullOrWhiteSpace(record.MutationBatchId)) |
| | 32 | 211 | | .ToLookup(record => record.MutationBatchId, StringComparer.Ordinal); |
| | 16 | 212 | | ILookup<string, ApplicationAuditCompletionOutboxEntry> completionBatches = completionEntries |
| | 12 | 213 | | .Where(entry => !string.IsNullOrWhiteSpace(entry.MutationBatchId)) |
| | 28 | 214 | | .ToLookup(entry => entry.MutationBatchId, StringComparer.Ordinal); |
| | | 215 | | |
| | 48 | 216 | | foreach (AuditRecord malformed in auditRecords.Where(record => string.IsNullOrWhiteSpace(record.MutationBatchId) |
| | | 217 | | { |
| | 0 | 218 | | Add(candidates, Candidate( |
| | 0 | 219 | | ApplicationAuditReconciliationReasonCodes.MalformedCorrelation, |
| | 0 | 220 | | ApplicationAuditReconciliationSeverities.Error, |
| | 0 | 221 | | $"missing-{malformed.Id:N}", |
| | 0 | 222 | | null, |
| | 0 | 223 | | "Retain the row, investigate the originating save path, and append remediation evidence.")); |
| | | 224 | | } |
| | | 225 | | |
| | 64 | 226 | | foreach (IGrouping<string, AuditRecord> batch in auditBatches) |
| | | 227 | | { |
| | 16 | 228 | | List<AuditRecord> records = [.. batch]; |
| | 16 | 229 | | List<ApplicationAuditCompletionOutboxEntry> completions = [.. completionBatches[batch.Key]]; |
| | 32 | 230 | | DateTime newestRecordUtc = records.Max(record => record.ModifiedOnUtc); |
| | | 231 | | |
| | 16 | 232 | | if (completions.Count == 0 && newestRecordUtc <= now - _options.CompletionGracePeriod) |
| | | 233 | | { |
| | 6 | 234 | | Add(candidates, Candidate( |
| | 6 | 235 | | ApplicationAuditReconciliationReasonCodes.MissingCompletion, |
| | 6 | 236 | | ApplicationAuditReconciliationSeverities.Critical, |
| | 6 | 237 | | batch.Key, |
| | 6 | 238 | | null, |
| | 6 | 239 | | "Preserve the audit batch, investigate transaction completion, and append an operator remediation re |
| | | 240 | | } |
| | | 241 | | |
| | 56 | 242 | | foreach (ApplicationAuditCompletionOutboxEntry completion in completions) |
| | | 243 | | { |
| | 12 | 244 | | if (completion.AuditRecordCount != records.Count) |
| | | 245 | | { |
| | 2 | 246 | | Add(candidates, Candidate( |
| | 2 | 247 | | ApplicationAuditReconciliationReasonCodes.AuditRecordCountMismatch, |
| | 2 | 248 | | ApplicationAuditReconciliationSeverities.Critical, |
| | 2 | 249 | | batch.Key, |
| | 2 | 250 | | completion.Destination, |
| | 2 | 251 | | "Do not rewrite audit rows; compare retained evidence with the originating transaction and docum |
| | | 252 | | } |
| | | 253 | | |
| | 12 | 254 | | ApplicationMutationAuditReceipt receipt = ToReceipt(completion); |
| | 12 | 255 | | if (!_manifestVerifier.Verify(receipt, records)) |
| | | 256 | | { |
| | 4 | 257 | | Add(candidates, Candidate( |
| | 4 | 258 | | ApplicationAuditReconciliationReasonCodes.ManifestVerificationFailed, |
| | 4 | 259 | | ApplicationAuditReconciliationSeverities.Critical, |
| | 4 | 260 | | batch.Key, |
| | 4 | 261 | | completion.Destination, |
| | 4 | 262 | | "Quarantine downstream use of the batch, preserve all records, and investigate unauthorized or i |
| | | 263 | | } |
| | | 264 | | } |
| | | 265 | | |
| | 32 | 266 | | if (records.Any(record => record.State == "Added" && string.IsNullOrWhiteSpace(record.KeyValues))) |
| | | 267 | | { |
| | 0 | 268 | | Add(candidates, Candidate( |
| | 0 | 269 | | ApplicationAuditReconciliationReasonCodes.IncompleteGeneratedValues, |
| | 0 | 270 | | ApplicationAuditReconciliationSeverities.Error, |
| | 0 | 271 | | batch.Key, |
| | 0 | 272 | | null, |
| | 0 | 273 | | "Verify generated keys in the business database and append remediation evidence without modifying th |
| | | 274 | | } |
| | | 275 | | |
| | 16 | 276 | | if (HasMalformedCorrelation(records)) |
| | | 277 | | { |
| | 0 | 278 | | Add(candidates, Candidate( |
| | 0 | 279 | | ApplicationAuditReconciliationReasonCodes.MalformedCorrelation, |
| | 0 | 280 | | ApplicationAuditReconciliationSeverities.Warning, |
| | 0 | 281 | | batch.Key, |
| | 0 | 282 | | null, |
| | 0 | 283 | | "Investigate inconsistent correlation metadata and preserve the original records as evidence.")); |
| | | 284 | | } |
| | | 285 | | } |
| | | 286 | | |
| | 52 | 287 | | foreach (IGrouping<string, ApplicationAuditCompletionOutboxEntry> batch in completionBatches) |
| | | 288 | | { |
| | 10 | 289 | | if (!auditBatches.Contains(batch.Key)) |
| | | 290 | | { |
| | 0 | 291 | | foreach (ApplicationAuditCompletionOutboxEntry completion in batch) |
| | | 292 | | { |
| | 0 | 293 | | Add(candidates, Candidate( |
| | 0 | 294 | | ApplicationAuditReconciliationReasonCodes.MissingAuditBatch, |
| | 0 | 295 | | ApplicationAuditReconciliationSeverities.Critical, |
| | 0 | 296 | | batch.Key, |
| | 0 | 297 | | completion.Destination, |
| | 0 | 298 | | "Preserve the completion record and investigate missing or externally stored audit evidence.")); |
| | | 299 | | } |
| | | 300 | | } |
| | | 301 | | } |
| | | 302 | | |
| | 36 | 303 | | foreach (IGrouping<(string MutationBatchId, string Destination), ApplicationAuditCompletionOutboxEntry> duplicat |
| | 28 | 304 | | completionEntries.GroupBy(entry => (entry.MutationBatchId, entry.Destination))) |
| | | 305 | | { |
| | 10 | 306 | | if (duplicate.Count() > 1) |
| | | 307 | | { |
| | 2 | 308 | | Add(candidates, Candidate( |
| | 2 | 309 | | ApplicationAuditReconciliationReasonCodes.DuplicateCompletion, |
| | 2 | 310 | | ApplicationAuditReconciliationSeverities.Critical, |
| | 2 | 311 | | duplicate.Key.MutationBatchId, |
| | 2 | 312 | | duplicate.Key.Destination, |
| | 2 | 313 | | "Preserve all records, stop dispatch for the destination, and investigate uniqueness or migration dr |
| | | 314 | | } |
| | | 315 | | } |
| | | 316 | | |
| | 56 | 317 | | foreach (ApplicationAuditCompletionOutboxEntry entry in completionEntries) |
| | | 318 | | { |
| | 12 | 319 | | AddDeliveryCandidate(candidates, entry, now); |
| | | 320 | | } |
| | | 321 | | |
| | 16 | 322 | | return [.. candidates.Values]; |
| | | 323 | | } |
| | | 324 | | |
| | | 325 | | private void AddDeliveryCandidate( |
| | | 326 | | IDictionary<string, ApplicationAuditReconciliationCandidate> candidates, |
| | | 327 | | ApplicationAuditCompletionOutboxEntry entry, |
| | | 328 | | DateTime now) |
| | | 329 | | { |
| | 12 | 330 | | if ((entry.Status == ApplicationAuditCompletionOutboxStatuses.Pending || |
| | 12 | 331 | | entry.Status == ApplicationAuditCompletionOutboxStatuses.Deferred) && |
| | 12 | 332 | | entry.CreatedUtc <= now - _options.StalePendingThreshold) |
| | | 333 | | { |
| | 2 | 334 | | Add(candidates, Candidate( |
| | 2 | 335 | | ApplicationAuditReconciliationReasonCodes.StalePending, |
| | 2 | 336 | | ApplicationAuditReconciliationSeverities.Warning, |
| | 2 | 337 | | entry.MutationBatchId, |
| | 2 | 338 | | entry.Destination, |
| | 2 | 339 | | "Verify dispatcher availability and destination registration before retrying delivery.")); |
| | | 340 | | } |
| | 10 | 341 | | else if (entry.Status == ApplicationAuditCompletionOutboxStatuses.RetryableFailure && |
| | 10 | 342 | | entry.NextAttemptUtc <= now - _options.StaleRetryReadyThreshold) |
| | | 343 | | { |
| | 0 | 344 | | Add(candidates, Candidate( |
| | 0 | 345 | | ApplicationAuditReconciliationReasonCodes.StaleRetryReady, |
| | 0 | 346 | | ApplicationAuditReconciliationSeverities.Error, |
| | 0 | 347 | | entry.MutationBatchId, |
| | 0 | 348 | | entry.Destination, |
| | 0 | 349 | | "Inspect destination availability and retry policy; preserve prior attempt diagnostics.")); |
| | | 350 | | } |
| | 10 | 351 | | else if (entry.Status == ApplicationAuditCompletionOutboxStatuses.Failed) |
| | | 352 | | { |
| | 0 | 353 | | Add(candidates, Candidate( |
| | 0 | 354 | | ApplicationAuditReconciliationReasonCodes.DeliveryFailed, |
| | 0 | 355 | | ApplicationAuditReconciliationSeverities.Error, |
| | 0 | 356 | | entry.MutationBatchId, |
| | 0 | 357 | | entry.Destination, |
| | 0 | 358 | | "Investigate the terminal delivery failure and append operator remediation evidence.")); |
| | | 359 | | } |
| | 10 | 360 | | else if (entry.Status == ApplicationAuditCompletionOutboxStatuses.DeadLettered) |
| | | 361 | | { |
| | 0 | 362 | | Add(candidates, Candidate( |
| | 0 | 363 | | ApplicationAuditReconciliationReasonCodes.DeadLettered, |
| | 0 | 364 | | ApplicationAuditReconciliationSeverities.Critical, |
| | 0 | 365 | | entry.MutationBatchId, |
| | 0 | 366 | | entry.Destination, |
| | 0 | 367 | | "Review the dead letter, preserve diagnostics, correct the destination, and explicitly requeue only unde |
| | | 368 | | } |
| | 10 | 369 | | } |
| | | 370 | | |
| | | 371 | | private async Task PersistCandidatesAsync( |
| | | 372 | | IReadOnlyCollection<ApplicationAuditReconciliationCandidate> candidates, |
| | | 373 | | IReadOnlyCollection<string> auditBatchIds, |
| | | 374 | | IReadOnlyCollection<ApplicationAuditCompletionOutboxEntry> completionEntries, |
| | | 375 | | DateTime now, |
| | | 376 | | CancellationToken cancellationToken) |
| | | 377 | | { |
| | | 378 | | string[] keys = [.. candidates.Select(candidate => candidate.FindingKey)]; |
| | 16 | 379 | | List<ApplicationAuditReconciliationFinding> existing = await _dbContext |
| | 16 | 380 | | .ApplicationAuditReconciliationFindings |
| | 16 | 381 | | .AsNoTracking() |
| | 16 | 382 | | .Where(finding => keys.Contains(finding.FindingKey)) |
| | 16 | 383 | | .ToListAsync(cancellationToken) |
| | 16 | 384 | | .ConfigureAwait(false); |
| | 16 | 385 | | var existingByKey = existing |
| | | 386 | | .ToDictionary(finding => finding.FindingKey, StringComparer.Ordinal); |
| | | 387 | | |
| | 64 | 388 | | foreach (ApplicationAuditReconciliationCandidate candidate in candidates) |
| | | 389 | | { |
| | 16 | 390 | | if (existingByKey.TryGetValue(candidate.FindingKey, out ApplicationAuditReconciliationFinding? finding)) |
| | | 391 | | { |
| | 2 | 392 | | await _dbContext.Database.ExecuteSqlInterpolatedAsync($$""" |
| | 2 | 393 | | UPDATE [ApplicationAuditReconciliationFindings] |
| | 2 | 394 | | SET [Severity] = {{candidate.Severity}}, |
| | 2 | 395 | | [Guidance] = {{candidate.Guidance}}, |
| | 2 | 396 | | [LastObservedUtc] = {{now}}, |
| | 2 | 397 | | [RemediationStatus] = {{ApplicationAuditReconciliationRemediationStatuses.Open}}, |
| | 2 | 398 | | [ResolvedUtc] = {{(DateTime?)null}}, |
| | 2 | 399 | | [ConcurrencyStamp] = {{Guid.NewGuid().ToString("N")}} |
| | 2 | 400 | | WHERE [Id] = {{finding.Id}} |
| | 2 | 401 | | """, cancellationToken).ConfigureAwait(false); |
| | | 402 | | } |
| | | 403 | | else |
| | | 404 | | { |
| | 14 | 405 | | var id = Guid.NewGuid(); |
| | 14 | 406 | | await _dbContext.Database.ExecuteSqlInterpolatedAsync($$""" |
| | 14 | 407 | | INSERT INTO [ApplicationAuditReconciliationFindings] |
| | 14 | 408 | | ([Id], [SchemaVersion], [FindingKey], [ReasonCode], [Severity], [MutationBatchId], [Destination] |
| | 14 | 409 | | VALUES |
| | 14 | 410 | | ({{id}}, {{ApplicationAuditReconciliationFinding.CurrentSchemaVersion}}, {{candidate.FindingKey} |
| | 14 | 411 | | """, cancellationToken).ConfigureAwait(false); |
| | | 412 | | } |
| | | 413 | | } |
| | | 414 | | |
| | 16 | 415 | | string[] scopeBatchIds = [.. auditBatchIds |
| | | 416 | | .Concat(completionEntries.Select(entry => entry.MutationBatchId)) |
| | | 417 | | .Where(batchId => !string.IsNullOrWhiteSpace(batchId)) |
| | 16 | 418 | | .Distinct(StringComparer.Ordinal)]; |
| | 16 | 419 | | string[] activeKeys = keys; |
| | | 420 | | |
| | 16 | 421 | | List<ApplicationAuditReconciliationFinding> resolved = await _dbContext |
| | 16 | 422 | | .ApplicationAuditReconciliationFindings |
| | 16 | 423 | | .AsNoTracking() |
| | 16 | 424 | | .Where(finding => scopeBatchIds.Contains(finding.MutationBatchId) && |
| | 16 | 425 | | finding.RemediationStatus != ApplicationAuditReconciliationRemediationStatuses.Resolved && |
| | 16 | 426 | | !activeKeys.Contains(finding.FindingKey)) |
| | 16 | 427 | | .ToListAsync(cancellationToken) |
| | 16 | 428 | | .ConfigureAwait(false); |
| | | 429 | | |
| | 32 | 430 | | foreach (ApplicationAuditReconciliationFinding finding in resolved) |
| | | 431 | | { |
| | 0 | 432 | | await _dbContext.Database.ExecuteSqlInterpolatedAsync($$""" |
| | 0 | 433 | | UPDATE [ApplicationAuditReconciliationFindings] |
| | 0 | 434 | | SET [RemediationStatus] = {{ApplicationAuditReconciliationRemediationStatuses.Resolved}}, |
| | 0 | 435 | | [ResolvedUtc] = {{now}}, |
| | 0 | 436 | | [ConcurrencyStamp] = {{Guid.NewGuid().ToString("N")}} |
| | 0 | 437 | | WHERE [Id] = {{finding.Id}} |
| | 0 | 438 | | """, cancellationToken).ConfigureAwait(false); |
| | | 439 | | } |
| | 16 | 440 | | } |
| | | 441 | | |
| | | 442 | | private async Task<ApplicationAuditReconciliationSummary> GetSummaryCoreAsync( |
| | | 443 | | DateTime? lastRunUtc, |
| | | 444 | | CancellationToken cancellationToken) |
| | | 445 | | { |
| | 16 | 446 | | IQueryable<ApplicationAuditReconciliationFinding> open = _dbContext |
| | 16 | 447 | | .ApplicationAuditReconciliationFindings |
| | 16 | 448 | | .AsNoTracking() |
| | 16 | 449 | | .Where(finding => finding.RemediationStatus != ApplicationAuditReconciliationRemediationStatuses.Resolved); |
| | | 450 | | |
| | 16 | 451 | | long openCount = await open.LongCountAsync(cancellationToken).ConfigureAwait(false); |
| | 16 | 452 | | long errorCount = await open.LongCountAsync( |
| | 16 | 453 | | finding => finding.Severity == ApplicationAuditReconciliationSeverities.Error, |
| | 16 | 454 | | cancellationToken).ConfigureAwait(false); |
| | 16 | 455 | | long criticalCount = await open.LongCountAsync( |
| | 16 | 456 | | finding => finding.Severity == ApplicationAuditReconciliationSeverities.Critical, |
| | 16 | 457 | | cancellationToken).ConfigureAwait(false); |
| | 16 | 458 | | long manifestFailures = await open.LongCountAsync( |
| | 16 | 459 | | finding => finding.ReasonCode == ApplicationAuditReconciliationReasonCodes.ManifestVerificationFailed, |
| | 16 | 460 | | cancellationToken).ConfigureAwait(false); |
| | 16 | 461 | | long missingCompletion = await open.LongCountAsync( |
| | 16 | 462 | | finding => finding.ReasonCode == ApplicationAuditReconciliationReasonCodes.MissingCompletion, |
| | 16 | 463 | | cancellationToken).ConfigureAwait(false); |
| | 16 | 464 | | long staleDelivery = await open.LongCountAsync( |
| | 16 | 465 | | finding => finding.ReasonCode == ApplicationAuditReconciliationReasonCodes.StalePending || |
| | 16 | 466 | | finding.ReasonCode == ApplicationAuditReconciliationReasonCodes.StaleRetryReady || |
| | 16 | 467 | | finding.ReasonCode == ApplicationAuditReconciliationReasonCodes.DeliveryFailed, |
| | 16 | 468 | | cancellationToken).ConfigureAwait(false); |
| | 16 | 469 | | long deadLetters = await open.LongCountAsync( |
| | 16 | 470 | | finding => finding.ReasonCode == ApplicationAuditReconciliationReasonCodes.DeadLettered, |
| | 16 | 471 | | cancellationToken).ConfigureAwait(false); |
| | | 472 | | |
| | 16 | 473 | | return new( |
| | 16 | 474 | | true, |
| | 16 | 475 | | lastRunUtc, |
| | 16 | 476 | | openCount, |
| | 16 | 477 | | errorCount, |
| | 16 | 478 | | criticalCount, |
| | 16 | 479 | | manifestFailures, |
| | 16 | 480 | | missingCompletion, |
| | 16 | 481 | | staleDelivery, |
| | 16 | 482 | | deadLetters); |
| | 16 | 483 | | } |
| | | 484 | | |
| | | 485 | | private static bool HasMalformedCorrelation(IReadOnlyCollection<AuditRecord> records) |
| | | 486 | | { |
| | 16 | 487 | | return records.Any(record => |
| | 16 | 488 | | IsWhitespaceOnly(record.OperationExecutionId) || |
| | 16 | 489 | | IsWhitespaceOnly(record.ExecutionAttemptId) || |
| | 16 | 490 | | IsWhitespaceOnly(record.DecisionAuditRecordId) || |
| | 16 | 491 | | IsWhitespaceOnly(record.CorrelationId) || |
| | 16 | 492 | | IsWhitespaceOnly(record.TraceId)) || |
| | 16 | 493 | | HasMultipleValues(records.Select(record => record.OperationExecutionId)) || |
| | 16 | 494 | | HasMultipleValues(records.Select(record => record.ExecutionAttemptId)) || |
| | 16 | 495 | | HasMultipleValues(records.Select(record => record.DecisionAuditRecordId)) || |
| | 16 | 496 | | HasMultipleValues(records.Select(record => record.CorrelationId)) || |
| | 32 | 497 | | HasMultipleValues(records.Select(record => record.TraceId)); |
| | | 498 | | } |
| | | 499 | | |
| | | 500 | | private static bool HasMultipleValues(IEnumerable<string?> values) |
| | | 501 | | { |
| | 80 | 502 | | return values |
| | 80 | 503 | | .Where(value => !string.IsNullOrWhiteSpace(value)) |
| | 0 | 504 | | .Select(value => value!.Trim()) |
| | 80 | 505 | | .Distinct(StringComparer.Ordinal) |
| | 80 | 506 | | .Skip(1) |
| | 80 | 507 | | .Any(); |
| | | 508 | | } |
| | | 509 | | |
| | | 510 | | private static bool IsWhitespaceOnly(string? value) |
| | | 511 | | { |
| | 80 | 512 | | return value is not null && string.IsNullOrWhiteSpace(value); |
| | | 513 | | } |
| | | 514 | | |
| | | 515 | | private static ApplicationMutationAuditReceipt ToReceipt(ApplicationAuditCompletionOutboxEntry entry) |
| | | 516 | | { |
| | 12 | 517 | | return new( |
| | 12 | 518 | | entry.MutationBatchId, |
| | 12 | 519 | | entry.AuditRecordCount, |
| | 12 | 520 | | entry.PersistenceOutcome, |
| | 12 | 521 | | new DateTimeOffset(DateTime.SpecifyKind(entry.ReceiptCompletedUtc, DateTimeKind.Utc)), |
| | 12 | 522 | | entry.MutationManifestHash, |
| | 12 | 523 | | entry.MutationManifestAlgorithm, |
| | 12 | 524 | | entry.MutationManifestSchemaVersion, |
| | 12 | 525 | | entry.OperationExecutionId, |
| | 12 | 526 | | entry.ExecutionAttemptId, |
| | 12 | 527 | | entry.DecisionAuditRecordId, |
| | 12 | 528 | | entry.CorrelationId, |
| | 12 | 529 | | entry.TraceId); |
| | | 530 | | } |
| | | 531 | | |
| | | 532 | | private static ApplicationAuditReconciliationCandidate Candidate( |
| | | 533 | | string reasonCode, |
| | | 534 | | string severity, |
| | | 535 | | string batchId, |
| | | 536 | | string? destination, |
| | | 537 | | string guidance) |
| | | 538 | | { |
| | 16 | 539 | | string normalizedBatchId = NormalizeRequired(batchId, 64, nameof(batchId)); |
| | 16 | 540 | | string? normalizedDestination = NormalizeOptional(destination, 128); |
| | 16 | 541 | | string identity = $"{reasonCode}\n{normalizedBatchId}\n{normalizedDestination}"; |
| | 16 | 542 | | string key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(identity))); |
| | 16 | 543 | | return new(key, reasonCode, severity, normalizedBatchId, normalizedDestination, guidance); |
| | | 544 | | } |
| | | 545 | | |
| | | 546 | | private static void Add( |
| | | 547 | | IDictionary<string, ApplicationAuditReconciliationCandidate> candidates, |
| | | 548 | | ApplicationAuditReconciliationCandidate candidate) |
| | | 549 | | { |
| | 16 | 550 | | candidates[candidate.FindingKey] = candidate; |
| | 16 | 551 | | } |
| | | 552 | | |
| | | 553 | | private static string NormalizeRequired(string value, int maximumLength, string parameterName) |
| | | 554 | | { |
| | 20 | 555 | | ArgumentException.ThrowIfNullOrWhiteSpace(value, parameterName); |
| | 20 | 556 | | string normalized = value.Trim(); |
| | 20 | 557 | | return normalized.Length <= maximumLength |
| | 20 | 558 | | ? normalized |
| | 20 | 559 | | : throw new ArgumentOutOfRangeException(parameterName, $"Value cannot exceed {maximumLength} characters."); |
| | | 560 | | } |
| | | 561 | | |
| | | 562 | | private static string? NormalizeOptional(string? value, int maximumLength) |
| | | 563 | | { |
| | 18 | 564 | | if (string.IsNullOrWhiteSpace(value)) |
| | | 565 | | { |
| | 6 | 566 | | return null; |
| | | 567 | | } |
| | | 568 | | |
| | 12 | 569 | | string normalized = value.Trim(); |
| | 12 | 570 | | return normalized.Length <= maximumLength |
| | 12 | 571 | | ? normalized |
| | 12 | 572 | | : normalized[..maximumLength]; |
| | | 573 | | } |
| | | 574 | | |
| | | 575 | | private DateTime UtcNow() |
| | | 576 | | { |
| | 18 | 577 | | return _timeProvider.GetUtcNow().UtcDateTime; |
| | | 578 | | } |
| | | 579 | | } |