Session 05 - System Design Case Studies

Case Study A: E-Commerce Order System

Requirements

• 500K orders/ngày, peak 2x vào khung giờ flash sale
• Inventory phải chính xác (không oversell)
• Payment flow reliable, idempotent
• Order status tracking real-time
• 99.9% availability

Architecture

Key Decision: Inventory Management

Vấn đề: 1000 người cùng mua sản phẩm còn 10 cái - làm sao không oversell?

Giải pháp: Optimistic Locking + Redis Atomic Decrement

csharp
public class InventoryService
{
    private readonly IDatabase _redis;
    private readonly InventoryDbContext _db;

    public async Task<bool> ReserveStockAsync(int productId, int quantity)
    {
        var stockKey = $"inventory:{productId}:stock";

        // Atomic decrement trong Redis - thread-safe, fast
        var remaining = await _redis.StringDecrementAsync(stockKey, quantity);

        if (remaining < 0)
        {
            // Rollback - không đủ hàng
            await _redis.StringIncrementAsync(stockKey, quantity);
            return false;
        }

        // Ghi reservation vào DB (async, eventual consistency)
        await _db.Reservations.AddAsync(new Reservation
        {
            ProductId = productId,
            Quantity = quantity,
            Status = ReservationStatus.Reserved,
            ExpiresAt = DateTime.UtcNow.AddMinutes(15) // Auto-release sau 15 phút
        });
        await _db.SaveChangesAsync();

        return true;
    }

    // Background job: release expired reservations
    public async Task ReleaseExpiredReservationsAsync()
    {
        var expired = await _db.Reservations
            .Where(r => r.Status == ReservationStatus.Reserved
                     && r.ExpiresAt < DateTime.UtcNow)
            .ToListAsync();

        foreach (var reservation in expired)
        {
            reservation.Status = ReservationStatus.Released;
            await _redis.StringIncrementAsync(
                $"inventory:{reservation.ProductId}:stock",
                reservation.Quantity);
        }
        await _db.SaveChangesAsync();
    }
}

Key Decision: Payment Flow (Idempotency)

csharp
public class PaymentService
{
    public async Task<PaymentResult> ProcessPaymentAsync(PaymentRequest request)
    {
        // Idempotency key ngăn double-charge
        var existing = await _db.Payments
            .FirstOrDefaultAsync(p => p.IdempotencyKey == request.IdempotencyKey);

        if (existing is not null)
            return new PaymentResult { Success = true, TransactionId = existing.TransactionId };

        // Saga pattern: từng bước có compensating action
        var transaction = await _db.Database.BeginTransactionAsync();
        try
        {
            // 1. Tạo payment record (PENDING)
            var payment = new Payment
            {
                IdempotencyKey = request.IdempotencyKey,
                Amount = request.Amount,
                Status = PaymentStatus.Pending
            };
            _db.Payments.Add(payment);
            await _db.SaveChangesAsync();

            // 2. Gọi payment gateway
            var gatewayResult = await _paymentGateway.ChargeAsync(request);

            // 3. Cập nhật status
            payment.Status = gatewayResult.Success
                ? PaymentStatus.Completed
                : PaymentStatus.Failed;
            payment.TransactionId = gatewayResult.TransactionId;
            await _db.SaveChangesAsync();

            await transaction.CommitAsync();
            return new PaymentResult
            {
                Success = gatewayResult.Success,
                TransactionId = gatewayResult.TransactionId
            };
        }
        catch
        {
            await transaction.RollbackAsync();
            throw;
        }
    }
}

Trade-offs

Scaling

• Order API: Horizontal scale 4-8 instances, stateless
• Inventory Redis: Premium tier, cluster mode cho flash sale
• SQL Database: Read replica cho query, primary cho write
• Service Bus: Auto-scaling, partition queue cho throughput

Case Study B: Real-Time Notification System

Requirements

• 100K concurrent users online
• Notification delivery < 3 giây
• Support: in-app (real-time), email, push notification (mobile)
• User có thể chọn channel nhận thông báo
• Notification history và read/unread status

Architecture

SignalR Hub Implementation

csharp
public class NotificationHub : Hub
{
    private readonly IConnectionMultiplexer _redis;

    public override async Task OnConnectedAsync()
    {
        var userId = Context.User?.FindFirst(ClaimTypes.NameIdentifier)?.Value;
        if (userId is null) return;

        // Track user connection trong Redis (support multiple server instances)
        var db = _redis.GetDatabase();
        await db.SetAddAsync($"user-connections:{userId}", Context.ConnectionId);
        await db.StringSetAsync($"connection-user:{Context.ConnectionId}", userId);

        // Join user-specific group
        await Groups.AddToGroupAsync(Context.ConnectionId, $"user:{userId}");

        await base.OnConnectedAsync();
    }

    public override async Task OnDisconnectedAsync(Exception? exception)
    {
        var db = _redis.GetDatabase();
        var userId = await db.StringGetAsync($"connection-user:{Context.ConnectionId}");

        if (userId.HasValue)
        {
            await db.SetRemoveAsync($"user-connections:{userId}", Context.ConnectionId);
            await db.KeyDeleteAsync($"connection-user:{Context.ConnectionId}");
        }

        await base.OnDisconnectedAsync(exception);
    }
}

Notification Router

csharp
public class NotificationRouter
{
    private readonly IHubContext<NotificationHub> _hubContext;
    private readonly IDatabase _redis;
    private readonly ServiceBusSender _emailSender;
    private readonly ServiceBusSender _pushSender;

    public async Task RouteNotificationAsync(NotificationEvent evt)
    {
        // 1. Lưu vào DB
        var notification = new Notification
        {
            UserId = evt.UserId,
            Title = evt.Title,
            Body = evt.Body,
            Type = evt.Type,
            IsRead = false,
            CreatedAt = DateTimeOffset.UtcNow
        };
        await _db.Notifications.AddAsync(notification);
        await _db.SaveChangesAsync();

        // 2. Lấy user preferences
        var prefs = await _db.NotificationPreferences
            .FirstOrDefaultAsync(p => p.UserId == evt.UserId);

        // 3. Real-time (SignalR) - luôn gửi nếu user online
        var isOnline = await _redis.SetLengthAsync($"user-connections:{evt.UserId}") > 0;
        if (isOnline)
        {
            await _hubContext.Clients.Group($"user:{evt.UserId}")
                .SendAsync("ReceiveNotification", new
                {
                    notification.Id,
                    notification.Title,
                    notification.Body,
                    notification.Type,
                    notification.CreatedAt
                });
        }

        // 4. Email - theo preference
        if (prefs?.EmailEnabled == true)
        {
            await _emailSender.SendMessageAsync(new ServiceBusMessage(
                JsonSerializer.SerializeToUtf8Bytes(new EmailMessage
                {
                    To = evt.UserEmail,
                    Subject = notification.Title,
                    Body = notification.Body
                })));
        }

        // 5. Push notification - nếu user offline hoặc preference bật
        if (!isOnline || prefs?.PushEnabled == true)
        {
            await _pushSender.SendMessageAsync(new ServiceBusMessage(
                JsonSerializer.SerializeToUtf8Bytes(new PushMessage
                {
                    UserId = evt.UserId,
                    Title = notification.Title,
                    Body = notification.Body
                })));
        }
    }
}

Trade-offs

Scaling

• SignalR: Azure SignalR Service (managed) - handle 100K+ connections, auto-scale
• Router Workers: Horizontal scale, partition by user ID hash
• Email/Push Workers: Scale independent theo queue depth
• Redis: Single instance đủ cho 100K users (< 100MB memory)

Case Study C: File Processing Pipeline

Requirements

• Upload file lên đến 500MB (Excel, CSV, PDF)
• Parse, validate, transform data
• Xử lý async, user track progress real-time
• Handle 1000 files/giờ peak
• Retry failed processing, dead-letter cho manual review

Architecture

Upload API

csharp
[HttpPost("upload")]
[RequestSizeLimit(500 * 1024 * 1024)] // 500MB
public async Task<IActionResult> UploadFile(IFormFile file)
{
    var jobId = Guid.NewGuid();
    var blobName = $"uploads/{jobId}/{file.FileName}";

    // 1. Upload trực tiếp lên Blob Storage (stream, không buffer toàn bộ vào memory)
    var blobClient = _blobContainer.GetBlobClient(blobName);
    await using var stream = file.OpenReadStream();
    await blobClient.UploadAsync(stream, new BlobUploadOptions
    {
        HttpHeaders = new BlobHttpHeaders { ContentType = file.ContentType },
        Metadata = new Dictionary<string, string>
        {
            ["originalName"] = file.FileName,
            ["uploadedBy"] = User.Identity!.Name!,
            ["jobId"] = jobId.ToString()
        }
    });

    // 2. Tạo processing job record
    await _db.ProcessingJobs.AddAsync(new ProcessingJob
    {
        Id = jobId,
        FileName = file.FileName,
        BlobPath = blobName,
        Status = ProcessingStatus.Uploaded,
        CreatedAt = DateTimeOffset.UtcNow
    });
    await _db.SaveChangesAsync();

    // 3. Enqueue để bắt đầu pipeline
    await _queueClient.SendMessageAsync(
        JsonSerializer.Serialize(new FileUploadedMessage
        {
            JobId = jobId,
            BlobPath = blobName,
            FileName = file.FileName
        }));

    return Accepted(new { jobId, status = "uploaded" });
}

Azure Function: Validate Step

csharp
public class ValidateFunction
{
    [Function("ValidateFile")]
    [QueueOutput("file-validated")]
    public async Task<FileValidatedMessage?> Run(
        [QueueTrigger("file-uploaded")] FileUploadedMessage message,
        FunctionContext context)
    {
        var logger = context.GetLogger<ValidateFunction>();
        await UpdateStatusAsync(message.JobId, ProcessingStatus.Validating);

        try
        {
            var blobClient = _blobContainer.GetBlobClient(message.BlobPath);
            await using var stream = await blobClient.OpenReadAsync();

            // Validate file format, size, content
            var validation = message.FileName.EndsWith(".csv")
                ? await ValidateCsvAsync(stream)
                : await ValidateExcelAsync(stream);

            if (!validation.IsValid)
            {
                await UpdateStatusAsync(message.JobId, ProcessingStatus.ValidationFailed,
                    string.Join("; ", validation.Errors));
                // Return null → message goes to poison queue sau max retries
                throw new ValidationException(string.Join("; ", validation.Errors));
            }

            await UpdateStatusAsync(message.JobId, ProcessingStatus.Validated);

            return new FileValidatedMessage
            {
                JobId = message.JobId,
                BlobPath = message.BlobPath,
                RowCount = validation.RowCount,
                Schema = validation.DetectedSchema
            };
        }
        catch (Exception ex) when (ex is not ValidationException)
        {
            logger.LogError(ex, "Unexpected error validating {JobId}", message.JobId);
            await UpdateStatusAsync(message.JobId, ProcessingStatus.Error, ex.Message);
            throw; // Retry via queue mechanism
        }
    }
}

Progress Tracking

csharp
// Client-side: track progress qua SignalR
public class ProcessingProgressHub : Hub
{
    public async Task SubscribeToJob(Guid jobId)
    {
        await Groups.AddToGroupAsync(Context.ConnectionId, $"job:{jobId}");

        // Gửi status hiện tại ngay lập tức
        var job = await _db.ProcessingJobs.FindAsync(jobId);
        if (job is not null)
        {
            await Clients.Caller.SendAsync("ProgressUpdate", new
            {
                job.Id,
                Status = job.Status.ToString(),
                job.ProgressPercent,
                job.ErrorMessage
            });
        }
    }
}

// Trong mỗi Function step: update progress
private async Task UpdateStatusAsync(
    Guid jobId, ProcessingStatus status, string? error = null, int? progress = null)
{
    var job = await _db.ProcessingJobs.FindAsync(jobId);
    if (job is null) return;

    job.Status = status;
    job.ErrorMessage = error;
    if (progress.HasValue) job.ProgressPercent = progress.Value;
    job.UpdatedAt = DateTimeOffset.UtcNow;
    await _db.SaveChangesAsync();

    // Push real-time update
    await _hubContext.Clients.Group($"job:{jobId}")
        .SendAsync("ProgressUpdate", new
        {
            job.Id,
            Status = status.ToString(),
            ProgressPercent = progress ?? job.ProgressPercent,
            ErrorMessage = error
        });
}

Trade-offs

Scaling

• Upload API: App Service 2-4 instances, rate limit uploads
• Azure Functions: Consumption plan, auto-scale 0 to N instances theo queue depth
• Blob Storage: Virtually unlimited, auto-partitioned
• Queue: Azure Storage Queue cho simplicity (65K messages visible), Service Bus nếu cần ordering
• Bottleneck thường gặp: DB writes khi load data - giải quyết bằng bulk insert (SqlBulkCopy) hoặc staging table