feat: Enhance network membership and withdrawal processing with user tracking and logging

2025-12-01 20:52:18 +03:30
parent 4aaf2247ff
commit 25fc73ae28
47 changed files with 9545 additions and 284 deletions
@@ -0,0 +1,230 @@
+using CMSMicroservice.Application.CommissionCQ.Commands.CalculateWeeklyBalances;
+using CMSMicroservice.Application.CommissionCQ.Commands.CalculateWeeklyCommissionPool;
+using CMSMicroservice.Application.CommissionCQ.Commands.ProcessUserPayouts;
+using CMSMicroservice.Application.Common.Interfaces;
+using CMSMicroservice.Domain.Entities;
+using CMSMicroservice.Domain.Enums;
+using MediatR;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Polly;
+
+namespace CMSMicroservice.Infrastructure.BackgroundJobs;
+
+/// <summary>
+/// Hangfire Job for weekly commission calculation
+/// Executes every Sunday at 00:05 (Cron: "5 0 * * 0")
+/// </summary>
+public class WeeklyCommissionJob
+{
+    private readonly IMediator _mediator;
+    private readonly ILogger<WeeklyCommissionJob> _logger;
+    private readonly IApplicationDbContext _context;
+    private readonly ResiliencePipeline _retryPipeline;
+
+    public WeeklyCommissionJob(
+        IMediator mediator,
+        ILogger<WeeklyCommissionJob> logger,
+        IApplicationDbContext context)
+    {
+        _mediator = mediator;
+        _logger = logger;
+        _context = context;
+
+        // Polly Retry: 3 attempts, exponential backoff (5min → 10min → 20min)
+        _retryPipeline = new ResiliencePipelineBuilder()
+            .AddRetry(new Polly.Retry.RetryStrategyOptions
+            {
+                MaxRetryAttempts = 3,
+                Delay = TimeSpan.FromMinutes(5),
+                BackoffType = Polly.DelayBackoffType.Exponential,
+                UseJitter = true,
+                OnRetry = args =>
+                {
+                    _logger.LogWarning(
+                        "⚠️ Retry attempt {AttemptNumber} after {Delay}ms delay. Exception: {ExceptionType}",
+                        args.AttemptNumber,
+                        args.RetryDelay.TotalMilliseconds,
+                        args.Outcome.Exception?.GetType().Name ?? "None");
+                    return ValueTask.CompletedTask;
+                }
+            })
+            .Build();
+    }
+
+    /// <summary>
+    /// Execute weekly commission calculation with retry logic
+    /// Called by Hangfire scheduler
+    /// </summary>
+    public async Task ExecuteAsync(CancellationToken cancellationToken = default)
+    {
+        var executionId = Guid.NewGuid();
+        var startTime = DateTime.UtcNow;
+
+        // Calculate for PREVIOUS week (completed week)
+        var previousWeek = DateTime.UtcNow.AddDays(-7);
+        var previousWeekNumber = GetWeekNumber(previousWeek);
+
+        _logger.LogInformation(
+            "🚀 [{ExecutionId}] Starting weekly commission calculation for {WeekNumber}",
+            executionId, previousWeekNumber);
+
+        // Create execution log entry
+        var log = new WorkerExecutionLog
+        {
+            ExecutionId = executionId,
+            WeekNumber = previousWeekNumber,
+            StartedAt = startTime,
+            Status = WorkerExecutionStatus.Running
+        };
+        _context.WorkerExecutionLogs.Add(log);
+        await _context.SaveChangesAsync(cancellationToken);
+
+        try
+        {
+            // Execute with retry pipeline
+            await _retryPipeline.ExecuteAsync(async ct =>
+            {
+                await ExecuteWeeklyCalculationAsync(executionId, previousWeekNumber, ct);
+            }, cancellationToken);
+
+            // Update log on success
+            var completedAt = DateTime.UtcNow;
+            var duration = completedAt - startTime;
+
+            log.Status = WorkerExecutionStatus.Success;
+            log.CompletedAt = completedAt;
+            log.DurationMs = (long)duration.TotalMilliseconds;
+
+            // Get counts from database
+            var balancesCount = await _context.NetworkWeeklyBalances
+                .CountAsync(x => x.WeekNumber == previousWeekNumber, cancellationToken);
+            var payoutsCount = await _context.UserCommissionPayouts
+                .CountAsync(x => x.WeekNumber == previousWeekNumber, cancellationToken);
+
+            log.ProcessedCount = balancesCount + payoutsCount;
+
+            await _context.SaveChangesAsync(cancellationToken);
+
+            _logger.LogInformation(
+                "✅ [{ExecutionId}] Completed successfully in {Duration}s | Balances: {BalancesCount}, Payouts: {PayoutsCount}",
+                executionId, duration.TotalSeconds, balancesCount, payoutsCount);
+        }
+        catch (Exception ex)
+        {
+            // Update log on failure
+            var completedAt = DateTime.UtcNow;
+            var duration = completedAt - startTime;
+
+            log.Status = WorkerExecutionStatus.Failed;
+            log.CompletedAt = completedAt;
+            log.DurationMs = (long)duration.TotalMilliseconds;
+            log.ErrorMessage = ex.Message;
+            log.ErrorStackTrace = ex.StackTrace;
+
+            await _context.SaveChangesAsync(cancellationToken);
+
+            _logger.LogError(ex,
+                "❌ [{ExecutionId}] Failed after {Duration}s: {ErrorMessage}",
+                executionId, duration.TotalSeconds, ex.Message);
+
+            throw; // Re-throw for Hangfire to mark job as failed
+        }
+    }
+
+    private async Task ExecuteWeeklyCalculationAsync(
+        Guid executionId,
+        string weekNumber,
+        CancellationToken cancellationToken)
+    {
+        // Check idempotency: Skip if already calculated
+        var existingPool = await _context.WeeklyCommissionPools
+            .FirstOrDefaultAsync(x => x.WeekNumber == weekNumber, cancellationToken);
+
+        if (existingPool != null && existingPool.IsCalculated)
+        {
+            _logger.LogWarning(
+                "⚠️ [{ExecutionId}] Week {WeekNumber} already calculated. Skipping.",
+                executionId, weekNumber);
+            return;
+        }
+
+        using var transaction = new System.Transactions.TransactionScope(
+            System.Transactions.TransactionScopeOption.Required,
+            new System.Transactions.TransactionOptions
+            {
+                IsolationLevel = System.Transactions.IsolationLevel.ReadCommitted,
+                Timeout = TimeSpan.FromMinutes(30)
+            },
+            System.Transactions.TransactionScopeAsyncFlowOption.Enabled);
+
+        try
+        {
+            // Step 1: Calculate user balances (Left/Right leg volumes)
+            _logger.LogInformation(
+                "📊 [{ExecutionId}] Step 1/3: Calculating weekly balances...",
+                executionId);
+
+            await _mediator.Send(new CalculateWeeklyBalancesCommand
+            {
+                WeekNumber = weekNumber,
+                ForceRecalculate = false
+            }, cancellationToken);
+
+            // Step 2: Calculate global commission pool
+            _logger.LogInformation(
+                "💰 [{ExecutionId}] Step 2/3: Calculating commission pool...",
+                executionId);
+
+            await _mediator.Send(new CalculateWeeklyCommissionPoolCommand
+            {
+                WeekNumber = weekNumber,
+                ForceRecalculate = false
+            }, cancellationToken);
+
+            // Step 3: Distribute commissions to users
+            _logger.LogInformation(
+                "💸 [{ExecutionId}] Step 3/3: Processing user payouts...",
+                executionId);
+
+            await _mediator.Send(new ProcessUserPayoutsCommand
+            {
+                WeekNumber = weekNumber,
+                ForceReprocess = false
+            }, cancellationToken);
+
+            transaction.Complete();
+
+            _logger.LogInformation(
+                "✅ [{ExecutionId}] All 3 steps completed successfully",
+                executionId);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex,
+                "❌ [{ExecutionId}] Transaction rolled back: {ErrorMessage}",
+                executionId, ex.Message);
+            throw;
+        }
+    }
+
+    /// <summary>
+    /// Get ISO 8601 week number (YYYY-Www format)
+    /// </summary>
+    private static string GetWeekNumber(DateTime date)
+    {
+        var calendar = System.Globalization.CultureInfo.InvariantCulture.Calendar;
+        var weekNumber = calendar.GetWeekOfYear(
+            date,
+            System.Globalization.CalendarWeekRule.FirstFourDayWeek,
+            DayOfWeek.Monday);
+
+        var year = date.Year;
+        if (weekNumber >= 52 && date.Month == 1)
+            year--;
+        else if (weekNumber == 1 && date.Month == 12)
+            year++;
+
+        return $"{year}-W{weekNumber:D2}";
+    }
+}
@@ -9,6 +9,9 @@ using CMSMicroservice.Application.CommissionCQ.Commands.CalculateWeeklyBalances;
 using CMSMicroservice.Application.CommissionCQ.Commands.CalculateWeeklyCommissionPool;
 using CMSMicroservice.Application.CommissionCQ.Commands.ProcessUserPayouts;
 using CMSMicroservice.Application.Common.Interfaces;
+using CMSMicroservice.Domain.Entities.Commission;
+using Polly;
+using Polly.Retry;

 namespace CMSMicroservice.Infrastructure.BackgroundJobs;

@@ -20,17 +23,35 @@ public class WeeklyNetworkCommissionWorker : BackgroundService
 {
    private readonly ILogger<WeeklyNetworkCommissionWorker> _logger;
    private readonly IServiceProvider _serviceProvider;
-    private readonly IAlertService _alertService;
    private Timer? _timer;
+    private readonly ResiliencePipeline _retryPipeline;

    public WeeklyNetworkCommissionWorker(
        ILogger<WeeklyNetworkCommissionWorker> logger,
-        IServiceProvider serviceProvider,
-        IAlertService alertService)
+        IServiceProvider serviceProvider)
    {
        _logger = logger;
        _serviceProvider = serviceProvider;
-        _alertService = alertService;
+        
+        // ایجاد Retry Policy با Exponential Backoff
+        _retryPipeline = new ResiliencePipelineBuilder()
+            .AddRetry(new RetryStrategyOptions
+            {
+                MaxRetryAttempts = 3,
+                Delay = TimeSpan.FromMinutes(5),
+                BackoffType = DelayBackoffType.Exponential,
+                UseJitter = true,
+                OnRetry = args =>
+                {
+                    _logger.LogWarning(
+                        "Retry attempt {AttemptNumber} after {Delay}ms due to: {Exception}",
+                        args.AttemptNumber,
+                        args.RetryDelay.TotalMilliseconds,
+                        args.Outcome.Exception?.Message);
+                    return ValueTask.CompletedTask;
+                }
+            })
+            .Build();
    }

    protected override Task ExecuteAsync(CancellationToken stoppingToken)
@@ -52,9 +73,11 @@ public class WeeklyNetworkCommissionWorker : BackgroundService

        _logger.LogInformation("Next execution scheduled for: {NextRun}", nextRunTime);

-        // تنظیم timer برای اجرا در زمان مشخص و تکرار هفتگی
+        // تنظیم timer برای اجرا در زمان مشخص و تکرار هفتگی با Retry
        _timer = new Timer(
-            callback: async _ => await ExecuteWeeklyCalculationAsync(stoppingToken),
+            callback: async _ => await _retryPipeline.ExecuteAsync(
+                async ct => await ExecuteWeeklyCalculationAsync(ct),
+                stoppingToken),
            state: null,
            dueTime: delay,
            period: TimeSpan.FromDays(7) // هر 7 روز یکبار
@@ -86,10 +109,12 @@ public class WeeklyNetworkCommissionWorker : BackgroundService
    private async Task ExecuteWeeklyCalculationAsync(CancellationToken cancellationToken)
    {
        var executionId = Guid.NewGuid();
-        var startTime = DateTime.Now; // استفاده از Local Time
-        _logger.LogInformation("=== Starting Weekly Commission Calculation [{ExecutionId}] at {Time} (Local Time) ===", 
+        var startTime = DateTime.UtcNow;
+        _logger.LogInformation("=== Starting Weekly Commission Calculation [{ExecutionId}] at {Time} (UTC) ===", 
            executionId, startTime);

+        WorkerExecutionLog? log = null;
+
        try
        {
            using var scope = _serviceProvider.CreateScope();
@@ -102,6 +127,19 @@ public class WeeklyNetworkCommissionWorker : BackgroundService

            _logger.LogInformation("Processing week: {WeekNumber}", previousWeekNumber);

+            // ایجاد Log
+            log = new WorkerExecutionLog
+            {
+                ExecutionId = executionId,
+                WeekNumber = previousWeekNumber,
+                StartedAt = startTime,
+                Status = WorkerExecutionStatus.Running,
+                ProcessedCount = 0,
+                ErrorCount = 0
+            };
+            await context.WorkerExecutionLogs.AddAsync(log, cancellationToken);
+            await context.SaveChangesAsync(cancellationToken);
+
            // ===== IDEMPOTENCY CHECK =====
            // بررسی اینکه آیا این هفته قبلاً محاسبه شده یا نه
            var existingPool = await context.WeeklyCommissionPools
@@ -113,6 +151,13 @@ public class WeeklyNetworkCommissionWorker : BackgroundService
                _logger.LogWarning(
                    "Week {WeekNumber} already calculated. Skipping execution [{ExecutionId}]",
                    previousWeekNumber, executionId);
+                
+                // Update log
+                log.Status = WorkerExecutionStatus.SuccessWithWarnings;
+                log.CompletedAt = DateTime.UtcNow;
+                log.DurationMs = (long)(log.CompletedAt.Value - log.StartedAt).TotalMilliseconds;
+                log.Details = "Week already calculated - skipped";
+                await context.SaveChangesAsync(cancellationToken);
                return;
            }

@@ -177,7 +222,20 @@ public class WeeklyNetworkCommissionWorker : BackgroundService
                // Commit Transaction
                transaction.Complete();

-                var duration = DateTime.Now - startTime; // محاسبه مدت زمان با Local Time
+                var completedAt = DateTime.UtcNow;
+                var duration = completedAt - startTime;
+                
+                // Update log - Success
+                if (log != null)
+                {
+                    log.Status = WorkerExecutionStatus.Success;
+                    log.CompletedAt = completedAt;
+                    log.DurationMs = (long)duration.TotalMilliseconds;
+                    log.ProcessedCount = balancesCalculated + payoutsProcessed;
+                    log.Details = $"Success: {balancesCalculated} balances, {payoutsProcessed} payouts, {balancesToExpire.Count} expired";
+                    await context.SaveChangesAsync(cancellationToken);
+                }
+
                _logger.LogInformation(
                    "=== Weekly Commission Calculation Completed Successfully [{ExecutionId}] ===" +
                    "\n  Week: {WeekNumber}" +
@@ -217,6 +275,8 @@ public class WeeklyNetworkCommissionWorker : BackgroundService
        }
        catch (Exception ex)
        {
+            var previousWeekNumber = GetPreviousWeekNumber();
+            
            _logger.LogCritical(ex, 
                "!!! CRITICAL ERROR in Weekly Commission Calculation [{ExecutionId}] !!!" +
                "\n  Week: {WeekNumber}" +
@@ -224,24 +284,47 @@ public class WeeklyNetworkCommissionWorker : BackgroundService
                "\n  StackTrace: {StackTrace}" +
                "\n  Please investigate immediately!", 
                executionId,
-                GetPreviousWeekNumber(),
+                previousWeekNumber,
                ex.Message,
                ex.StackTrace);
            
+            // Update log - Failed
+            if (log != null)
+            {
+                try
+                {
+                    using var errorScope = _serviceProvider.CreateScope();
+                    var context = errorScope.ServiceProvider.GetRequiredService<IApplicationDbContext>();
+                    
+                    log.Status = WorkerExecutionStatus.Failed;
+                    log.CompletedAt = DateTime.UtcNow;
+                    log.DurationMs = (long)(log.CompletedAt.Value - log.StartedAt).TotalMilliseconds;
+                    log.ErrorCount = 1;
+                    log.ErrorMessage = ex.Message;
+                    log.ErrorStackTrace = ex.StackTrace;
+                    
+                    await context.SaveChangesAsync(cancellationToken);
+                }
+                catch (Exception logEx)
+                {
+                    _logger.LogError(logEx, "Failed to update error log");
+                }
+            }
+            
            // ===== ERROR HANDLING & ALERTING =====
            // در محیط production باید Alert/Notification ارسال شود
            
-            using var errorScope = _serviceProvider.CreateScope();
-            var alertService = errorScope.ServiceProvider.GetRequiredService<IAlertService>();
+            using var alertScope = _serviceProvider.CreateScope();
+            var alertService = alertScope.ServiceProvider.GetRequiredService<IAlertService>();
            
            await alertService.SendCriticalAlertAsync(
                "Weekly Commission Worker Failed",
-                $"Worker execution {executionId} failed for week {GetPreviousWeekNumber()}",
+                $"Worker execution {executionId} failed for week {previousWeekNumber}. Will retry with exponential backoff.",
                ex,
                cancellationToken);
            
-            // TODO: Retry logic با exponential backoff
-            // await RetryWithExponentialBackoff(() => ExecuteWeeklyCalculationAsync(cancellationToken));
+            // Retry با Polly - اگر همچنان fail کند exception throw می‌شود
+            throw;
        }
    }