diff --git a/.gitignore b/.gitignore index ce892922..938d4906 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,11 @@ BenchmarkDotNet.Artifacts/ project.lock.json project.fragment.lock.json artifacts/ +**/bin/ +**/obj/ +**/Debug/ +**/Release/ +**/publish/ # ASP.NET Scaffolding ScaffoldingReadMe.txt diff --git a/Client.Wasm/Components/StudentCard.razor b/Client.Wasm/Components/StudentCard.razor index 661f1181..6e17a1e8 100644 --- a/Client.Wasm/Components/StudentCard.razor +++ b/Client.Wasm/Components/StudentCard.razor @@ -4,10 +4,10 @@ - Номер №X "Название лабораторной" - Вариант №Х "Название варианта" - Выполнена Фамилией Именем 65ХХ - Ссылка на форк + Номер №3 "«Интеграционное тестирование» - Реализация файлового сервиса и объектного хранилища, интеграционное тестирование бекенда" + Вариант №41 "Кредитная заявка " + Выполнена Кадниковым Егором 6513 + Ссылка на форк diff --git a/Client.Wasm/wwwroot/appsettings.json b/Client.Wasm/wwwroot/appsettings.json index d1fe7ab3..2272e7d2 100644 --- a/Client.Wasm/wwwroot/appsettings.json +++ b/Client.Wasm/wwwroot/appsettings.json @@ -1,10 +1,10 @@ { - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "AllowedHosts": "*", - "BaseAddress": "" -} + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "BaseAddress": "https://localhost:7124/api/orders" +} \ No newline at end of file diff --git a/CloudDevelopment.AppHost/AppHost.cs b/CloudDevelopment.AppHost/AppHost.cs new file mode 100644 index 00000000..e3674a21 --- /dev/null +++ b/CloudDevelopment.AppHost/AppHost.cs @@ -0,0 +1,71 @@ +using Amazon; +using Amazon.CDK.AWS.Servicecatalog; +using Aspire.Hosting.LocalStack.Container; +using LocalStack.Client.Enums; +using Microsoft.Extensions.Configuration; + +var builder = DistributedApplication.CreateBuilder(args); + +var ports = builder.Configuration.GetSection("ApiService:Ports").Get() + ?? throw new InvalidOperationException("ApiService:Ports is not configured."); + +var cache = builder.AddRedis("credit-order-cache") + .WithRedisInsight(containerName: "credit-order-insight"); + +var awsConfig = builder.AddAWSSDKConfig() + .WithProfile("default") + .WithRegion(RegionEndpoint.EUCentral1); + +var localstack = builder + .AddLocalStack("credid-order-localstack", awsConfig: awsConfig, configureContainer: container => + { + container.Lifetime = ContainerLifetime.Session; + container.DebugLevel = 1; + container.LogLevel = LocalStackLogLevel.Debug; + container.Port = 4566; + container.AdditionalEnvironmentVariables + .Add("DEBUG", "1"); + container.AdditionalEnvironmentVariables + .Add("SNS_CERT_URL_HOST", "sns.eu-central-1.amazonaws.com"); + }); + +var cloudFormationTemplate = "CloudFormation/sqs-s3.yml"; +var awsResources = builder.AddAWSCloudFormationTemplate("resources", cloudFormationTemplate, "credit-order") + .WithReference(awsConfig); + +var gateway = builder.AddProject("gateway"); +for (var i = 0; i < ports.Length; i++) +{ + var httpsPort = ports[i]; + var httpPort = ports[i] - 1000; + + var generator = builder.AddProject($"generator-r{i + 1}", launchProfileName: null) + .WithReference(cache, "RedisCache") + .WithHttpEndpoint(httpPort) + .WithReference(awsResources) + .WithHttpsEndpoint(httpsPort) + .WaitFor(cache) + .WaitFor(awsResources); + + gateway.WaitFor(generator); +} + +builder.AddProject("client-wasm") + .WaitFor(gateway); + +var storage = builder.AddProject("service-storage") + .WithHttpEndpoint(5444, name: "storege-http") + .WithReference(awsResources) + .WithEnvironment("Settings__MessageBroker", "SQS") + .WithEnvironment("Settings__S3Hosting", "Minio") + .WaitFor(awsResources); + +var minio = builder.AddMinioContainer("credit-order-minio"); + +storage.WithEnvironment("AWS__Resources__MinioBucketName", "credit-order-bucket") + .WithReference(minio) + .WaitFor(minio); + +builder.UseLocalStack(localstack); + +builder.Build().Run(); diff --git a/CloudDevelopment.AppHost/CloudDevelopment.AppHost.csproj b/CloudDevelopment.AppHost/CloudDevelopment.AppHost.csproj new file mode 100644 index 00000000..b1f213e2 --- /dev/null +++ b/CloudDevelopment.AppHost/CloudDevelopment.AppHost.csproj @@ -0,0 +1,37 @@ + + + + + + Exe + net8.0 + enable + enable + 2f419979-4dcb-43a3-bf86-3b5579f994b9 + + + + + + + + + + + + + + + + + + + Always + + + + + + + + diff --git a/CloudDevelopment.AppHost/CloudFormation/sqs-s3.yml b/CloudDevelopment.AppHost/CloudFormation/sqs-s3.yml new file mode 100644 index 00000000..b637a964 --- /dev/null +++ b/CloudDevelopment.AppHost/CloudFormation/sqs-s3.yml @@ -0,0 +1,62 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Cloud formation template for credit order project' + +Parameters: + BucketName: + Type: String + Description: Name for the S3 bucket + Default: 'credit-order-bucket' + + QueueName: + Type: String + Description: Name for the SQS queue + Default: 'credit-order-queue' + +Resources: + CreditOrderBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref BucketName + VersioningConfiguration: + Status: Suspended + Tags: + - Key: Name + Value: !Ref BucketName + - Key: Environment + Value: Sample + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + + CreditOrderQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: !Ref QueueName + VisibilityTimeout: 30 + MessageRetentionPeriod: 345600 + DelaySeconds: 0 + ReceiveMessageWaitTimeSeconds: 0 + Tags: + - Key: Name + Value: !Ref QueueName + - Key: Environment + Value: Sample + +Outputs: + S3BucketName: + Description: Name of the S3 bucket + Value: !Ref CreditOrderBucket + + S3BucketArn: + Description: ARN of the S3 bucket + Value: !GetAtt CreditOrderBucket.Arn + + SQSQueueName: + Description: Name of the SQS queue + Value: !GetAtt CreditOrderQueue.QueueName + + SQSQueueArn: + Description: ARN of the SQS queue + Value: !GetAtt CreditOrderQueue.Arn \ No newline at end of file diff --git a/CloudDevelopment.AppHost/Properties/launchSettings.json b/CloudDevelopment.AppHost/Properties/launchSettings.json new file mode 100644 index 00000000..b5c6a54c --- /dev/null +++ b/CloudDevelopment.AppHost/Properties/launchSettings.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17129;http://localhost:15221", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21101", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22255" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:15221", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19083", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20274" + } + } + } +} diff --git a/CloudDevelopment.AppHost/appsettings.Development.json b/CloudDevelopment.AppHost/appsettings.Development.json new file mode 100644 index 00000000..0c208ae9 --- /dev/null +++ b/CloudDevelopment.AppHost/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/CloudDevelopment.AppHost/appsettings.json b/CloudDevelopment.AppHost/appsettings.json new file mode 100644 index 00000000..71bf8376 --- /dev/null +++ b/CloudDevelopment.AppHost/appsettings.json @@ -0,0 +1,8 @@ +{ + "ApiService": { + "Ports": [ 7241, 7242, 7243, 7244, 7245 ] + }, + "LocalStack": { + "UseLocalStack": true + } +} \ No newline at end of file diff --git a/CloudDevelopment.ServiceDefaults/CloudDevelopment.ServiceDefaults.csproj b/CloudDevelopment.ServiceDefaults/CloudDevelopment.ServiceDefaults.csproj new file mode 100644 index 00000000..1b6e209a --- /dev/null +++ b/CloudDevelopment.ServiceDefaults/CloudDevelopment.ServiceDefaults.csproj @@ -0,0 +1,22 @@ + + + + net8.0 + enable + enable + true + + + + + + + + + + + + + + + diff --git a/CloudDevelopment.ServiceDefaults/Extensions.cs b/CloudDevelopment.ServiceDefaults/Extensions.cs new file mode 100644 index 00000000..8b1a2b03 --- /dev/null +++ b/CloudDevelopment.ServiceDefaults/Extensions.cs @@ -0,0 +1,128 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.ServiceDiscovery; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +namespace CloudDevelopment.ServiceDefaults; + +// Adds common Aspire services: service discovery, resilience, health checks, and OpenTelemetry. +// This project should be referenced by each service project in your solution. +// To learn more about using this project, see https://aka.ms/dotnet/aspire/service-defaults +public static class Extensions +{ + private const string HealthEndpointPath = "/health"; + private const string AlivenessEndpointPath = "/alive"; + + public static TBuilder AddServiceDefaults(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + builder.ConfigureOpenTelemetry(); + + builder.AddDefaultHealthChecks(); + + builder.Services.AddServiceDiscovery(); + + builder.Services.ConfigureHttpClientDefaults(http => + { + // Turn on resilience by default + http.AddStandardResilienceHandler(); + + // Turn on service discovery by default + http.AddServiceDiscovery(); + }); + + // Uncomment the following to restrict the allowed schemes for service discovery. + // builder.Services.Configure(options => + // { + // options.AllowedSchemes = ["https"]; + // }); + + return builder; + } + + public static TBuilder ConfigureOpenTelemetry(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + builder.Logging.AddOpenTelemetry(logging => + { + logging.IncludeFormattedMessage = true; + logging.IncludeScopes = true; + }); + + builder.Services.AddOpenTelemetry() + .WithMetrics(metrics => + { + metrics.AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddRuntimeInstrumentation(); + }) + .WithTracing(tracing => + { + tracing.AddSource(builder.Environment.ApplicationName) + .AddAspNetCoreInstrumentation(tracing => + // Exclude health check requests from tracing + tracing.Filter = context => + !context.Request.Path.StartsWithSegments(HealthEndpointPath) + && !context.Request.Path.StartsWithSegments(AlivenessEndpointPath) + ) + // Uncomment the following line to enable gRPC instrumentation (requires the OpenTelemetry.Instrumentation.GrpcNetClient package) + //.AddGrpcClientInstrumentation() + .AddHttpClientInstrumentation(); + }); + + builder.AddOpenTelemetryExporters(); + + return builder; + } + + private static TBuilder AddOpenTelemetryExporters(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + var useOtlpExporter = !string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]); + + if (useOtlpExporter) + { + builder.Services.AddOpenTelemetry().UseOtlpExporter(); + } + + // Uncomment the following lines to enable the Azure Monitor exporter (requires the Azure.Monitor.OpenTelemetry.AspNetCore package) + //if (!string.IsNullOrEmpty(builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"])) + //{ + // builder.Services.AddOpenTelemetry() + // .UseAzureMonitor(); + //} + + return builder; + } + + public static TBuilder AddDefaultHealthChecks(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + builder.Services.AddHealthChecks() + // Add a default liveness check to ensure app is responsive + .AddCheck("self", () => HealthCheckResult.Healthy(), ["live"]); + + return builder; + } + + public static WebApplication MapDefaultEndpoints(this WebApplication app) + { + // Adding health checks endpoints to applications in non-development environments has security implications. + // See https://aka.ms/dotnet/aspire/healthchecks for details before enabling these endpoints in non-development environments. + if (app.Environment.IsDevelopment()) + { + // All health checks must pass for app to be considered ready to accept traffic after starting + app.MapHealthChecks(HealthEndpointPath); + + // Only health checks tagged with the "live" tag must pass for app to be considered alive + app.MapHealthChecks(AlivenessEndpointPath, new HealthCheckOptions + { + Predicate = r => r.Tags.Contains("live") + }); + } + + return app; + } +} diff --git a/CloudDevelopment.sln b/CloudDevelopment.sln index cb48241d..d143ef73 100644 --- a/CloudDevelopment.sln +++ b/CloudDevelopment.sln @@ -5,6 +5,18 @@ VisualStudioVersion = 17.14.36811.4 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Client.Wasm", "Client.Wasm\Client.Wasm.csproj", "{AE7EEA74-2FE0-136F-D797-854FD87E022A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Service.Api", "Generator\Service.Api.csproj", "{5F162047-71C4-A730-10F2-8456E4D1F966}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CloudDevelopment.AppHost", "CloudDevelopment.AppHost\CloudDevelopment.AppHost.csproj", "{359B77C3-1D6B-4E58-A926-C907812424A8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CloudDevelopment.ServiceDefaults", "CloudDevelopment.ServiceDefaults\CloudDevelopment.ServiceDefaults.csproj", "{DC017A15-5E73-C618-2A78-CD0D64478DC9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Api.Gateway", "CreditOrder.Gateway\Api.Gateway.csproj", "{8A14F5A2-48C3-1C30-0A17-D7579F6795C9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Service.Storage", "Service.Storage\Service.Storage.csproj", "{8A73C6A0-9E6F-1B4F-CC95-75E08F6D247B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntegrationTests", "IntegrationTests\IntegrationTests.csproj", "{0E29B01D-8437-41A8-B709-E22CF31AA194}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -15,6 +27,30 @@ Global {AE7EEA74-2FE0-136F-D797-854FD87E022A}.Debug|Any CPU.Build.0 = Debug|Any CPU {AE7EEA74-2FE0-136F-D797-854FD87E022A}.Release|Any CPU.ActiveCfg = Release|Any CPU {AE7EEA74-2FE0-136F-D797-854FD87E022A}.Release|Any CPU.Build.0 = Release|Any CPU + {5F162047-71C4-A730-10F2-8456E4D1F966}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5F162047-71C4-A730-10F2-8456E4D1F966}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F162047-71C4-A730-10F2-8456E4D1F966}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5F162047-71C4-A730-10F2-8456E4D1F966}.Release|Any CPU.Build.0 = Release|Any CPU + {359B77C3-1D6B-4E58-A926-C907812424A8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {359B77C3-1D6B-4E58-A926-C907812424A8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {359B77C3-1D6B-4E58-A926-C907812424A8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {359B77C3-1D6B-4E58-A926-C907812424A8}.Release|Any CPU.Build.0 = Release|Any CPU + {DC017A15-5E73-C618-2A78-CD0D64478DC9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DC017A15-5E73-C618-2A78-CD0D64478DC9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DC017A15-5E73-C618-2A78-CD0D64478DC9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DC017A15-5E73-C618-2A78-CD0D64478DC9}.Release|Any CPU.Build.0 = Release|Any CPU + {8A14F5A2-48C3-1C30-0A17-D7579F6795C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A14F5A2-48C3-1C30-0A17-D7579F6795C9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A14F5A2-48C3-1C30-0A17-D7579F6795C9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A14F5A2-48C3-1C30-0A17-D7579F6795C9}.Release|Any CPU.Build.0 = Release|Any CPU + {8A73C6A0-9E6F-1B4F-CC95-75E08F6D247B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A73C6A0-9E6F-1B4F-CC95-75E08F6D247B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A73C6A0-9E6F-1B4F-CC95-75E08F6D247B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A73C6A0-9E6F-1B4F-CC95-75E08F6D247B}.Release|Any CPU.Build.0 = Release|Any CPU + {0E29B01D-8437-41A8-B709-E22CF31AA194}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0E29B01D-8437-41A8-B709-E22CF31AA194}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E29B01D-8437-41A8-B709-E22CF31AA194}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0E29B01D-8437-41A8-B709-E22CF31AA194}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/CreditOrder.Gateway/Api.Gateway.csproj b/CreditOrder.Gateway/Api.Gateway.csproj new file mode 100644 index 00000000..f0acd112 --- /dev/null +++ b/CreditOrder.Gateway/Api.Gateway.csproj @@ -0,0 +1,17 @@ + + + + net8.0 + enable + enable + + + + + + + + + + + diff --git a/CreditOrder.Gateway/LoadBalancers/WeightedRandomLoadBalancer.cs b/CreditOrder.Gateway/LoadBalancers/WeightedRandomLoadBalancer.cs new file mode 100644 index 00000000..85e44087 --- /dev/null +++ b/CreditOrder.Gateway/LoadBalancers/WeightedRandomLoadBalancer.cs @@ -0,0 +1,113 @@ +using Api.Gateway.Models; +using Ocelot.LoadBalancer.Interfaces; +using Ocelot.Responses; +using Ocelot.Values; + +namespace Api.Gateway.LoadBalancers; + +/// +/// Балансировщик нагрузки для Ocelot, работает по алгоритму weighted random. +/// +/// Вес каждой реплики задаётся в конфигурации через секцию ReplicaWeights. +/// +/// Чем больше вес реплики, тем выше вероятность, что запрос будет направлен именно на неt. +/// +public sealed class WeightedRandomLoadBalancer(IConfiguration configuration, + Func>> _services) : ILoadBalancer +{ + /// + /// Конфигурация весов реплик downstream-сервисов. + /// + private readonly List _weights = configuration.GetSection("ReplicaWeights").Get>() ?? []; + + /// + /// Тип балансировщика нагрузки. + /// Используется Ocelot для сопоставления с конфигурацией + /// LoadBalancerOptions.Type. + /// + public string Type => nameof(WeightedRandomLoadBalancer); + + /// + /// lookup весов по паре (Host, Port). + /// + private readonly Dictionary<(string Host, int Port), int> _weightsByEndpoint = + (configuration.GetSection("ReplicaWeights").Get>() ?? []) + .ToDictionary( + w => (w.Host.ToLowerInvariant(), w.Port), + w => w.Weight); + + /// + /// Выбирает downstream-сервис для обработки запроса. + /// Если веса для сервисов не заданы, используется первый доступный сервис. + /// + /// + /// HTTP-контекст текущего запроса. + /// В данном балансировщике не используется. + /// + /// + /// Объект , + /// содержащий выбранный downstream-сервис. + /// + /// + /// Выбрасывается, если список downstream-сервисов пуст. + /// + public async Task> LeaseAsync(HttpContext _) + { + var services = await _services(); + + var candidates = new List<(Service Service, int Weight)>(); + + foreach (var service in services) + { + var key = ( + service.HostAndPort.DownstreamHost.ToLowerInvariant(), + service.HostAndPort.DownstreamPort + ); + + if (_weightsByEndpoint.TryGetValue(key, out var serviceWeight)) + { + candidates.Add((service, serviceWeight)); + } + } + + if (candidates.Count == 0) + { + var fallback = services.FirstOrDefault(); + if (fallback is null) + { + throw new InvalidOperationException("No downstream services configured."); + } + + return await Task.FromResult>( + new OkResponse(fallback.HostAndPort)); + } + + var totalWeight = candidates.Sum(x => x.Weight); + var resSum = Random.Shared.Next(1, totalWeight + 1); + + var weight = 0; + foreach (var candidate in candidates) + { + weight += candidate.Weight; + if (resSum <= weight) + { + return await Task.FromResult>( + new OkResponse(candidate.Service.HostAndPort)); + } + } + + return await Task.FromResult>( + new OkResponse(candidates[^1].Service.HostAndPort)); + } + + /// + /// Освобождает ранее выделенный сервис. + /// + /// В данном балансировщике метод не используется, + /// так как выбор сервиса происходит без удержания состояния. + /// + /// Адрес сервиса. + public void Release(ServiceHostAndPort _) + { + } +} \ No newline at end of file diff --git a/CreditOrder.Gateway/Models/ReplicaWeight.cs b/CreditOrder.Gateway/Models/ReplicaWeight.cs new file mode 100644 index 00000000..64ad078e --- /dev/null +++ b/CreditOrder.Gateway/Models/ReplicaWeight.cs @@ -0,0 +1,21 @@ +namespace Api.Gateway.Models; + +/// +/// Модель конфигурации реплики downstream-сервиса, используемая балансировщиком нагрузки шлюза. +/// +public class ReplicaWeight +{ + /// + /// Хост сервиса-реплики. + /// + public string Host { get; set; } = string.Empty; + /// + /// Порт, на котором работает реплика сервиса. + /// + public int Port { get; set; } + /// + /// Вес реплики для алгоритма балансировки нагрузки. + /// Чем больше значение, тем чаще на эту реплику будут направляться запросы. + /// + public int Weight { get; set; } +} diff --git a/CreditOrder.Gateway/Program.cs b/CreditOrder.Gateway/Program.cs new file mode 100644 index 00000000..364291d0 --- /dev/null +++ b/CreditOrder.Gateway/Program.cs @@ -0,0 +1,25 @@ +using Api.Gateway.LoadBalancers; +using Ocelot.DependencyInjection; +using Ocelot.LoadBalancer.Interfaces; +using Ocelot.Middleware; + +var builder = WebApplication.CreateBuilder(args); + +builder.Configuration.AddJsonFile("ocelot.json", optional: false, reloadOnChange: true); + +builder.Services + .AddOcelot(builder.Configuration) + .AddCustomLoadBalancer((serviceProvider, _, discoveryProvider) => + { + var configuration = serviceProvider.GetRequiredService(); + + return new WeightedRandomLoadBalancer( + configuration, + discoveryProvider!.GetAsync); + }); + +builder.Configuration.AddOcelot(); + +var app = builder.Build(); +await app.UseOcelot(); +await app.RunAsync(); diff --git a/CreditOrder.Gateway/Properties/launchSettings.json b/CreditOrder.Gateway/Properties/launchSettings.json new file mode 100644 index 00000000..c07b0a5c --- /dev/null +++ b/CreditOrder.Gateway/Properties/launchSettings.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:43281", + "sslPort": 44361 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5265", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:7124;http://localhost:5265", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/CreditOrder.Gateway/appsettings.Development.json b/CreditOrder.Gateway/appsettings.Development.json new file mode 100644 index 00000000..0c208ae9 --- /dev/null +++ b/CreditOrder.Gateway/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/CreditOrder.Gateway/appsettings.json b/CreditOrder.Gateway/appsettings.json new file mode 100644 index 00000000..85428df6 --- /dev/null +++ b/CreditOrder.Gateway/appsettings.json @@ -0,0 +1,29 @@ +{ + "ReplicaWeights": [ + { + "Host": "localhost", + "Port": 7241, + "Weight": 1 + }, + { + "Host": "localhost", + "Port": 7242, + "Weight": 2 + }, + { + "Host": "localhost", + "Port": 7243, + "Weight": 3 + }, + { + "Host": "localhost", + "Port": 7244, + "Weight": 4 + }, + { + "Host": "localhost", + "Port": 7245, + "Weight": 5 + } + ] +} \ No newline at end of file diff --git a/CreditOrder.Gateway/ocelot.json b/CreditOrder.Gateway/ocelot.json new file mode 100644 index 00000000..04fb670b --- /dev/null +++ b/CreditOrder.Gateway/ocelot.json @@ -0,0 +1,38 @@ +{ + "Routes": [ + { + "UpstreamPathTemplate": "/api/orders", + "UpstreamHttpMethod": [ "Get" ], + "DownstreamPathTemplate": "/credit-orders", + "DownstreamScheme": "https", + "DownstreamHostAndPorts": [ + { + "Host": "localhost", + "Port": 7241 + }, + { + "Host": "localhost", + "Port": 7242 + }, + { + "Host": "localhost", + "Port": 7243 + }, + { + "Host": "localhost", + "Port": 7244 + }, + { + "Host": "localhost", + "Port": 7245 + } + ], + "LoadBalancerOptions": { + "Type": "WeightedRandomLoadBalancer" + } + } + ], + "GlobalConfiguration": { + "BaseUrl": "https://localhost:7124" + } +} \ No newline at end of file diff --git a/Generator/Controllers/CreditOrderController.cs b/Generator/Controllers/CreditOrderController.cs new file mode 100644 index 00000000..cb483a94 --- /dev/null +++ b/Generator/Controllers/CreditOrderController.cs @@ -0,0 +1,37 @@ +using Service.Api.Dto; +using Service.Api.Services; +using Microsoft.AspNetCore.Mvc; + +namespace Service.Api.Controllers; + +/// +/// HTTP API для получения кредитной заявки по идентификатору. +/// Использует для получения данных (кэш + генерация). +/// +[ApiController] +[Route("credit-orders")] +public class CreditOrderController ( + CreditOrderService service, + ILogger logger + ) : ControllerBase +{ + + /// + /// Возвращает кредитную заявку по из query string. + /// + /// Идентификатор заявки (должен быть больше 0). + /// Токен отмены запроса. + /// DTO кредитной заявки. + /// Заявка успешно получена. + /// Некорректный идентификатор (id <= 0). + [HttpGet] + public async Task> Get([FromQuery] int id, CancellationToken ct) + { + logger.LogInformation("HTTP GET /credit-orders requested: {OrderId}", id); + if (id <= 0) + return BadRequest("id must be greater than 0"); + var order = await service.GetByIdAsync(id, ct); + logger.LogInformation("HTTP GET /credit-orders completed: {OrderId} {Status}", order.Id, order.OrderStatus); + return Ok(order); + } +} diff --git a/Generator/Dto/CreditOrderDto.cs b/Generator/Dto/CreditOrderDto.cs new file mode 100644 index 00000000..50879f52 --- /dev/null +++ b/Generator/Dto/CreditOrderDto.cs @@ -0,0 +1,41 @@ +namespace Service.Api.Dto; + +/// +/// DTO кредитной заявки, возвращаемый HTTP API. +/// +public class CreditOrderDto +{ + /// Идентификатор заявки. + public int Id { get; set; } + + /// Тип кредита (например: Потребительский, Ипотека). + public string CreditType { get; set; } = ""; + + /// Запрошенная сумма кредита. + public decimal RequestedSum { get; set; } + + /// Срок кредита в месяцах. + public int MonthsDuration { get; set; } + + /// Процентная ставка (годовых). + public double InterestRate { get; set; } + + /// Дата подачи заявки. + public DateOnly FilingDate { get; set; } + + /// Признак необходимости страховки. + public bool IsInsuranceNeeded { get; set; } + + /// Статус заявки: Новая / В обработке / Одобрена / Отклонена. + public string OrderStatus { get; set; } = ""; + + /// + /// Дата принятия решения. Заполняется только для конечных статусов (например, Одобрена/Отклонена). + /// + public DateOnly? DecisionDate { get; set; } + + /// + /// Одобренная сумма. Заполняется только при статусе "Одобрена". + /// + public decimal? ApprovedSum { get; set; } +} diff --git a/Generator/Program.cs b/Generator/Program.cs new file mode 100644 index 00000000..aa0b326a --- /dev/null +++ b/Generator/Program.cs @@ -0,0 +1,62 @@ +using Amazon.SQS; +using CloudDevelopment.ServiceDefaults; +using LocalStack.Client.Extensions; +using Service.Api.Services; + +var builder = WebApplication.CreateBuilder(args); + +builder.AddServiceDefaults(); + +builder.Services.AddControllers(); +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(); + +builder.Services.AddCors(options => +{ + options.AddDefaultPolicy(policy => + policy.SetIsOriginAllowed(origin => + { + try + { + var uri = new Uri(origin); + return uri.Host == "localhost"; + } + catch + { + return false; + } + }) + .WithMethods("GET") + .AllowAnyHeader()); +}); + +builder.Services.AddScoped(); +builder.Services.AddLocalStack(builder.Configuration); +builder.Services.AddAwsService(); + +builder.AddRedisDistributedCache(connectionName: "RedisCache"); + +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +var app = builder.Build(); + +app.UseCors(); + +app.MapDefaultEndpoints(); + +if (app.Environment.IsDevelopment()) +{ + app.UseSwagger(); + app.UseSwaggerUI(); +} + +app.UseHttpsRedirection(); + +app.UseAuthorization(); + +app.UseCors("wasm"); + +app.MapControllers(); + +app.Run(); diff --git a/Generator/Properties/launchSettings.json b/Generator/Properties/launchSettings.json new file mode 100644 index 00000000..7d04f4fe --- /dev/null +++ b/Generator/Properties/launchSettings.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:6127", + "sslPort": 44325 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/Generator/Service.Api.csproj b/Generator/Service.Api.csproj new file mode 100644 index 00000000..5f5fcb3b --- /dev/null +++ b/Generator/Service.Api.csproj @@ -0,0 +1,23 @@ + + + + net8.0 + enable + enable + + + + + + + + + + + + + + + + + diff --git a/Generator/Service.Api.csproj.user b/Generator/Service.Api.csproj.user new file mode 100644 index 00000000..9ff5820a --- /dev/null +++ b/Generator/Service.Api.csproj.user @@ -0,0 +1,6 @@ + + + + https + + \ No newline at end of file diff --git a/Generator/Services/CreditOrderGenerator.cs b/Generator/Services/CreditOrderGenerator.cs new file mode 100644 index 00000000..82d08c57 --- /dev/null +++ b/Generator/Services/CreditOrderGenerator.cs @@ -0,0 +1,49 @@ +using Bogus; +using Service.Api.Dto; + +namespace Service.Api.Services; + +/// +/// Генератор псевдослучайных кредитных заявок для демо/тестирования. +/// Включает простые зависимости между статусом и полями решения. +/// +public class CreditOrderGenerator +{ + private static readonly string[] _сreditTypes = { "Потребительский", "Ипотека", "Автокредит", "Микрозайм" }; + private static readonly string[] _statuses = { "Новая", "В обработке", "Одобрена", "Отклонена" }; + + /// + /// Генерирует кредитную заявку для заданного с реалистичными полями. + /// + /// Идентификатор заявки. + /// Сгенерированная заявка. + public CreditOrderDto Generate(int id) + { + var faker = new Faker("ru") + .RuleFor(x => x.Id, _ => id) + .RuleFor(x => x.CreditType, f => f.PickRandom(_сreditTypes)) + .RuleFor(x => x.RequestedSum, f => Math.Round(f.Finance.Amount(1_000_000m, 100_000_000m), 2)) + .RuleFor(x => x.MonthsDuration, f => f.Random.Int(1, 360)) + .RuleFor(x => x.InterestRate, f => Math.Round(f.Random.Double(15.6, 20.0), 2)) + .RuleFor(x => x.FilingDate, f => DateOnly.FromDateTime(f.Date.Past(2))) + .RuleFor(x => x.IsInsuranceNeeded, f => f.Random.Bool()) + .RuleFor(x => x.OrderStatus, f => f.PickRandom(_statuses)) + .RuleFor(x => x.DecisionDate, _ => null) + .RuleFor(x => x.ApprovedSum, _ => null) + .RuleFor(x => x.DecisionDate, (f, o) => + o.OrderStatus is "Одобрена" or "Отклонена" + ? o.FilingDate.AddDays(f.Random.Int(0, 31)) + : null) + .RuleFor(x => x.ApprovedSum, (f, o) => + { + if (o.OrderStatus is not "Одобрена") + return null; + + var k = f.Random.Double(0.6, 1.0); + var approved = o.RequestedSum * (decimal)k; + return Math.Round(approved, 2); + }); + + return faker.Generate(); + } +} diff --git a/Generator/Services/CreditOrderService.cs b/Generator/Services/CreditOrderService.cs new file mode 100644 index 00000000..d0818b8e --- /dev/null +++ b/Generator/Services/CreditOrderService.cs @@ -0,0 +1,111 @@ +using Amazon.SQS; +using Microsoft.Extensions.Caching.Distributed; +using Service.Api.Dto; +using System.Text.Json; + +namespace Service.Api.Services; + +/// +/// Сервис получения кредитной заявки по идентификатору. +/// Сначала пытается вернуть данные из распределённого кэша, при отсутствии данных в кэше — генерирует заявку и кэширует результат. +/// +public class CreditOrderService( + IDistributedCache cache, + CreditOrderGenerator generator, + IConfiguration cfg, + SqsProducerService sqsProducer, + ILogger logger + ) +{ + + private static readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.Web); + + /// + /// Возвращает заявку по : + /// 1) читает из кэша по ключу credit-order:{id}; + /// 2) при отсутствии данных в кэше генерирует через ; + /// 3) сохраняет в кэш с TTL (AbsoluteExpirationRelativeToNow). + /// 4) отправляет сообщение в очередь. + /// + /// Идентификатор заявки (должен быть больше 0). + /// Токен отмены. + /// DTO заявки. + /// Если <= 0. + /// Если запрос был отменён. + public async Task GetByIdAsync(int id, CancellationToken ct) + { + var ttlSeconds = cfg.GetValue("CreditOrderCache:TtlSeconds", 300); + if (ttlSeconds <= 0) ttlSeconds = 300; + + var cacheKey = BuildCacheKey(id); + + try + { + var cachedJson = await cache.GetStringAsync(cacheKey, ct); + if (!string.IsNullOrWhiteSpace(cachedJson)) + { + var cached = JsonSerializer.Deserialize(cachedJson, _jsonOptions); + if (cached is not null) + { + logger.LogInformation("Cache HIT: {CacheKey} {OrderId}", cacheKey, id); + return cached; + } + + logger.LogWarning("Cache DESERIALIZE FAIL: {CacheKey} {OrderId}", cacheKey, id); + } + else + { + logger.LogInformation("Cache MISS: {CacheKey} {OrderId}", cacheKey, id); + } + } + catch (OperationCanceledException) + { + logger.LogInformation("Request canceled: {CacheKey} {OrderId}", cacheKey, id); + throw; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Cache READ FAIL: {CacheKey} {OrderId}", cacheKey, id); + } + + var order = generator.Generate(id); + + try + { + var cacheTtl = new DistributedCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(ttlSeconds) + }; + var json = JsonSerializer.Serialize(order, _jsonOptions); + await cache.SetStringAsync(cacheKey, json, cacheTtl, ct); + logger.LogInformation("Cache SET: {CacheKey} {OrderId}", cacheKey, id); + } + catch (OperationCanceledException) + { + logger.LogInformation("Request canceled: {CacheKey} {OrderId}", cacheKey, id); + throw; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Cache WRITE FAIL: {CacheKey} {OrderId}", cacheKey, id); + } + try + { + logger.LogInformation("Message publishing: {OrderId}", order.Id); + await sqsProducer.SendMessage(order); + logger.LogInformation("Message published: {OrderId}", order.Id); + } + catch (AmazonSQSException ex) + { + logger.LogError(ex, "SQS error while sending message {OrderId}", order.Id); + throw; + } + return order; + } + + /// + /// Формирует ключ кэша для заявки по идентификатору. + /// Формат: credit-order:{id}. + /// + private static string BuildCacheKey(int id) => $"credit-order:{id}"; +} diff --git a/Generator/Services/SqsProducerService.cs b/Generator/Services/SqsProducerService.cs new file mode 100644 index 00000000..efe98867 --- /dev/null +++ b/Generator/Services/SqsProducerService.cs @@ -0,0 +1,40 @@ +using Amazon.SQS; +using Service.Api.Dto; +using System.Net; +using System.Text.Json; + +namespace Service.Api.Services; + +/// +/// Служба для отправки сообщений в SQS +/// +/// Клиент SQS +/// Конфигурация +/// Логгер +public class SqsProducerService(IAmazonSQS client, IConfiguration configuration, ILogger logger) +{ + private readonly string _queueName = configuration["AWS:Resources:SQSQueueName"] + ?? throw new KeyNotFoundException("SQS queue link was not found in configuration"); + + /// + public async Task SendMessage(CreditOrderDto creditOrder) + { + try + { + var json = JsonSerializer.Serialize(creditOrder); + + var queueUrlResponse = await client.GetQueueUrlAsync(_queueName); + var queueUrl = queueUrlResponse.QueueUrl; + + var responce = await client.SendMessageAsync(queueUrl, json); + if (responce.HttpStatusCode == HttpStatusCode.OK) + logger.LogInformation("Land plot {id} was sent to sink via SQS", creditOrder.Id); + else + throw new Exception($"SQS returned {responce.HttpStatusCode}"); + } + catch (Exception ex) + { + logger.LogError(ex, "Unable to send cridit order through SQS queue"); + } + } +} diff --git a/Generator/appsettings.Development.json b/Generator/appsettings.Development.json new file mode 100644 index 00000000..0c208ae9 --- /dev/null +++ b/Generator/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/Generator/appsettings.json b/Generator/appsettings.json new file mode 100644 index 00000000..5490bdb0 --- /dev/null +++ b/Generator/appsettings.json @@ -0,0 +1,13 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "CreditOrderCache": { + "Enabled": true, + "TtlSeconds": 60 + }, + "AllowedHosts": "*" +} diff --git a/IntegrationTests/IntegrationTests.cs b/IntegrationTests/IntegrationTests.cs new file mode 100644 index 00000000..d23f197f --- /dev/null +++ b/IntegrationTests/IntegrationTests.cs @@ -0,0 +1,76 @@ +using Aspire.Hosting; +using Microsoft.Extensions.Logging; +using Service.Api.Dto; +using System.Net.Http.Json; +using System.Text.Json; +using Xunit.Abstractions; + +namespace IntegrationTests; + +/// +/// Интеграционные тесты для проверки микросервисного пайплайна +/// +/// Служба журналирования юнит-тестов +public class IntegrationTests(ITestOutputHelper output) : IAsyncLifetime +{ + private DistributedApplication? _app; + private CancellationToken _cancellationToken; + + /// + public async Task InitializeAsync() + { + _cancellationToken = CancellationToken.None; + var builder = await DistributedApplicationTestingBuilder.CreateAsync(_cancellationToken); + builder.Configuration["DcpPublisher:RandomizePorts"] = "false"; + builder.Services.AddLogging(logging => + { + logging.AddXUnit(output); + logging.SetMinimumLevel(LogLevel.Debug); + logging.AddFilter("Aspire.Hosting.Dcp", LogLevel.Debug); + logging.AddFilter("Aspire.Hosting", LogLevel.Debug); + }); + + builder.Environment.EnvironmentName = "Development"; + _app = await builder.BuildAsync(_cancellationToken); + await _app.StartAsync(_cancellationToken); + } + + /// + /// Проверяет, что вызов гейтвея: + /// + /// В ответ отправляет сгенерированный ЗУ + /// Сериализует ЗУ в S3 хранилище + /// Проверяет, что данные из предыдущих пунктов идентичны + /// + /// + [Fact] + public async Task TestPipeline() + { + var random = new Random(); + var id = random.Next(1, 100); + using var gatewayClient = _app.CreateHttpClient("gateway", "http"); + using var gatewayResponse = await gatewayClient!.GetAsync($"/api/orders?id={id}"); + var api = await gatewayResponse.Content.ReadFromJsonAsync(cancellationToken: _cancellationToken); + + await Task.Delay(5000); + using var sinkClient = _app.CreateHttpClient("service-storage", "http"); + using var listResponse = await sinkClient!.GetAsync($"/api/s3"); + var ppList = await listResponse.Content.ReadFromJsonAsync>(cancellationToken: _cancellationToken); + using var s3Response = await sinkClient!.GetAsync($"/api/s3/credit-order_{id}.json"); + var s3 = await s3Response.Content.ReadFromJsonAsync(cancellationToken: _cancellationToken); + + Assert.NotNull(ppList); + Assert.Single(ppList); + Assert.NotNull(api); + Assert.NotNull(s3); + Assert.Equal(id, s3.Id); + Assert.Equivalent(api, s3); + } + + /// + public async Task DisposeAsync() + { + await _app!.StopAsync(); + await _app.DisposeAsync(); + } +} diff --git a/IntegrationTests/IntegrationTests.csproj b/IntegrationTests/IntegrationTests.csproj new file mode 100644 index 00000000..c1770f85 --- /dev/null +++ b/IntegrationTests/IntegrationTests.csproj @@ -0,0 +1,33 @@ + + + + net8.0 + enable + enable + false + true + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Service.Storage/Controllers/S3StorageController.cs b/Service.Storage/Controllers/S3StorageController.cs new file mode 100644 index 00000000..29fda622 --- /dev/null +++ b/Service.Storage/Controllers/S3StorageController.cs @@ -0,0 +1,67 @@ +using Microsoft.AspNetCore.Mvc; +using Service.Storage.Storage; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Nodes; +using System.Threading.Tasks; + +namespace Service.Storage.Controllers; + +/// +/// Контроллер для взаимодейсвия с S3 +/// +/// Служба для работы с S3 +/// Логгер +[ApiController] +[Route("api/s3")] +public class S3StorageController(S3MinioService s3Service, ILogger logger) : ControllerBase +{ + /// + /// Метод для получения списка хранящихся в S3 файлов + /// + /// Список с ключами файлов + [HttpGet] + [ProducesResponseType(200)] + [ProducesResponseType(500)] + public async Task>> ListFiles() + { + logger.LogInformation("Method {method} of {controller} was called", nameof(ListFiles), nameof(S3StorageController)); + try + { + var list = await s3Service.GetFileList(); + logger.LogInformation("Got a list of {count} files from bucket", list.Count); + return Ok(list); + } + catch (Exception ex) + { + logger.LogError(ex, "Exception occured during {method} of {controller}", nameof(ListFiles), nameof(S3StorageController)); + return BadRequest(ex); + } + } + + /// + /// Получает строковое представление хранящегося в S3 документа + /// + /// Ключ файла + /// Строковое представление файла + [HttpGet("{key}")] + [ProducesResponseType(200)] + [ProducesResponseType(500)] + public async Task> GetFile(string key) + { + logger.LogInformation("Method {method} of {controller} was called", nameof(GetFile), nameof(S3StorageController)); + try + { + var node = await s3Service.DownloadFile(key); + logger.LogInformation("Received json of {size} bytes", Encoding.UTF8.GetByteCount(node.ToJsonString())); + return Ok(node); + } + catch (Exception ex) + { + logger.LogError(ex, "Exception occured during {method} of {controller}", nameof(GetFile), nameof(S3StorageController)); + return BadRequest(ex); + } + } +} diff --git a/Service.Storage/Messaging/SqsConsumerService.cs b/Service.Storage/Messaging/SqsConsumerService.cs new file mode 100644 index 00000000..c14ab5d3 --- /dev/null +++ b/Service.Storage/Messaging/SqsConsumerService.cs @@ -0,0 +1,78 @@ +using Amazon.SQS; +using Amazon.SQS.Model; +using Service.Storage.Storage; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Service.Storage.Messaging; + +/// +/// Клиентская служба для приема сообщений из очереди SQS +/// +/// Клиент SQS +/// Фабрика контекста +/// Конфигурация +/// Логгер +internal class SqsConsumerService(IAmazonSQS sqsClient, + IServiceScopeFactory scopeFactory, + IConfiguration configuration, + ILogger logger) : BackgroundService +{ + private readonly string _queueName = configuration["AWS:Resources:SQSQueueName"] + ?? throw new KeyNotFoundException("SQS queue name was not found in configuration"); + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation("SQS consumer service started."); + + var queueUrlResponse = await sqsClient.GetQueueUrlAsync(_queueName, stoppingToken); + var queueUrl = queueUrlResponse.QueueUrl; + + while (!stoppingToken.IsCancellationRequested) + { + var response = await sqsClient.ReceiveMessageAsync( + new ReceiveMessageRequest + { + QueueUrl = queueUrl, + MaxNumberOfMessages = 10, + WaitTimeSeconds = 5 + }, stoppingToken); + + if (response == null) + { + logger.LogWarning("Received null from {queue}", _queueName); + continue; + } + + logger.LogInformation("Received {count} messages", response!.Messages?.Count ?? 0); + + if (response.Messages != null) + { + + foreach (var message in response.Messages) + { + try + { + logger.LogInformation("Processing message: {messageId}", message.MessageId); + + using var scope = scopeFactory.CreateScope(); + var s3Service = scope.ServiceProvider.GetRequiredService(); + await s3Service.UploadFile(message.Body); + + _ = await sqsClient.DeleteMessageAsync(queueUrl, message.ReceiptHandle, stoppingToken); + } + catch (Exception ex) + { + logger.LogError(ex, "Error processing message: {messageId}", message.MessageId); + continue; + } + } + logger.LogInformation("Batch of {count} messages processed", response.Messages.Count); + } + } + } +} diff --git a/Service.Storage/Program.cs b/Service.Storage/Program.cs new file mode 100644 index 00000000..0c340604 --- /dev/null +++ b/Service.Storage/Program.cs @@ -0,0 +1,27 @@ +using Service.Storage; +using CloudDevelopment.ServiceDefaults; +using System.Reflection; + +var builder = WebApplication.CreateBuilder(args); +builder.AddServiceDefaults(); + +builder.Services.AddControllers(); +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(options => +{ + var assembly = Assembly.GetExecutingAssembly(); + options.IncludeXmlComments(Path.Combine(AppContext.BaseDirectory, $"{assembly.GetName().Name}.xml")); +}); + +builder.AddConsumer(); +builder.AddS3(); + +var app = builder.Build(); +await app.UseS3(); +if (app.Environment.IsDevelopment()) +{ + app.UseSwagger(); + app.UseSwaggerUI(); +} +app.MapControllers(); +app.Run(); \ No newline at end of file diff --git a/Service.Storage/Properties/launchSettings.json b/Service.Storage/Properties/launchSettings.json new file mode 100644 index 00000000..10617fcb --- /dev/null +++ b/Service.Storage/Properties/launchSettings.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "http://localhost:5280", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7099;http://localhost:5280", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} \ No newline at end of file diff --git a/Service.Storage/Service.Storage.csproj b/Service.Storage/Service.Storage.csproj new file mode 100644 index 00000000..c708c7a9 --- /dev/null +++ b/Service.Storage/Service.Storage.csproj @@ -0,0 +1,24 @@ + + + + net8.0 + enable + enable + dotnet-Service.Storage-25720c3a-f495-4948-ba5d-74e7ae702072 + Linux + true + + + + + + + + + + + + + + + diff --git a/Service.Storage/Storage/S3MinioService.cs b/Service.Storage/Storage/S3MinioService.cs new file mode 100644 index 00000000..b64ee966 --- /dev/null +++ b/Service.Storage/Storage/S3MinioService.cs @@ -0,0 +1,129 @@ +using System.Net; +using System.Text; +using System.Text.Json.Nodes; +using Minio; +using Minio.DataModel.Args; + +namespace Service.Storage.Storage; + +/// +/// Cлужба для манипуляции файлами в объектном хранилище +/// +/// S3 клиент +/// Конфигурация +/// Логер +public class S3MinioService(IMinioClient client, IConfiguration configuration, ILogger logger) +{ + private readonly string _bucketName = configuration["AWS:Resources:MinioBucketName"] + ?? throw new KeyNotFoundException("S3 bucket name was not found in configuration"); + + /// + public async Task> GetFileList() + { + var list = new List(); + var request = new ListObjectsArgs() + .WithBucket(_bucketName) + .WithPrefix("") + .WithRecursive(true); + logger.LogInformation("Began listing files in {bucket}", _bucketName); + var responseList = client.ListObjectsEnumAsync(request); + + if (responseList == null) + logger.LogWarning("Received null response from {bucket}", _bucketName); + + await foreach (var response in responseList!) + list.Add(response.Key); + return list; + } + + /// + public async Task UploadFile(string fileData) + { + var rootNode = JsonNode.Parse(fileData) ?? throw new ArgumentException("Passed string is not a valid JSON"); + var id = rootNode["Id"]?.GetValue() ?? throw new ArgumentException("Passed JSON has invalid structure"); + + var bytes = Encoding.UTF8.GetBytes(fileData); + using var stream = new MemoryStream(bytes); + stream.Seek(0, SeekOrigin.Begin); + + logger.LogInformation("Began uploading credit order {file} onto {bucket}", id, _bucketName); + var request = new PutObjectArgs() + .WithBucket(_bucketName) + .WithStreamData(stream) + .WithObjectSize(bytes.Length) + .WithObject($"credit-order_{id}.json"); + + var response = await client.PutObjectAsync(request); + + if (response.ResponseStatusCode != HttpStatusCode.OK) + { + logger.LogError("Failed to upload credit order {file}: {code}", id, response.ResponseStatusCode); + return false; + } + logger.LogInformation("Finished uploading credit order {file} to {bucket}", id, _bucketName); + return true; + } + + /// + public async Task DownloadFile(string key) + { + logger.LogInformation("Began downloading {file} from {bucket}", key, _bucketName); + + try + { + var memoryStream = new MemoryStream(); + + var request = new GetObjectArgs() + .WithBucket(_bucketName) + .WithObject(key) + .WithCallbackStream(async (stream, cancellationToken) => + { + await stream.CopyToAsync(memoryStream, cancellationToken); + memoryStream.Seek(0, SeekOrigin.Begin); + }); + + var response = await client.GetObjectAsync(request); + + if (response == null) + { + logger.LogError("Failed to download {file}", key); + throw new InvalidOperationException($"Error occurred downloading {key} - object is null"); + } + using var reader = new StreamReader(memoryStream, Encoding.UTF8); + return JsonNode.Parse(reader.ReadToEnd()) ?? throw new InvalidOperationException($"Downloaded document is not a valid JSON"); + } + catch (Exception ex) + { + logger.LogError(ex, "Exception occurred during {file} downloading ", key); + throw; + } + } + + /// + public async Task EnsureBucketExists() + { + logger.LogInformation("Checking whether {bucket} exists", _bucketName); + try + { + var request = new BucketExistsArgs() + .WithBucket(_bucketName); + + var exists = await client.BucketExistsAsync(request); + if (!exists) + { + + logger.LogInformation("Creating {bucket}", _bucketName); + var createRequest = new MakeBucketArgs() + .WithBucket(_bucketName); + await client.MakeBucketAsync(createRequest); + return; + } + logger.LogInformation("{bucket} exists", _bucketName); + } + catch (Exception ex) + { + logger.LogError(ex, "Unhandled exception occurred during {bucket} check", _bucketName); + throw; + } + } +} diff --git a/Service.Storage/WebApplicationBuilderExtensions.cs b/Service.Storage/WebApplicationBuilderExtensions.cs new file mode 100644 index 00000000..3cd504f7 --- /dev/null +++ b/Service.Storage/WebApplicationBuilderExtensions.cs @@ -0,0 +1,39 @@ +using Amazon.SQS; +using Service.Storage.Messaging; +using Service.Storage.Storage; +using LocalStack.Client.Extensions; + +namespace Service.Storage; + +/// +/// Экстеншен для добавления различных служб в DI в зависимости от конфигурации приложения +/// +internal static class WebApplicationBuilderExtensions +{ + /// + /// Регистрирует клиентские службы для работы с брокером сообщений + /// + /// Билдер + /// Билдер + /// Если настройки не найдены + public static WebApplicationBuilder AddConsumer(this WebApplicationBuilder builder) + { + builder.Services.AddLocalStack(builder.Configuration); + builder.Services.AddHostedService(); + builder.Services.AddAwsService(); + return builder; + } + + /// + /// Регистрирует клиентские службы для работы с объектным хранилищем + /// + /// Билдер + /// Билдер + /// Если настройки не найдены + public static WebApplicationBuilder AddS3(this WebApplicationBuilder builder) + { + builder.AddMinioClient("credit-order-minio"); + builder.Services.AddScoped(); + return builder; + } +} diff --git a/Service.Storage/WebApplicationExtensions.cs b/Service.Storage/WebApplicationExtensions.cs new file mode 100644 index 00000000..3a4731ee --- /dev/null +++ b/Service.Storage/WebApplicationExtensions.cs @@ -0,0 +1,24 @@ +using Microsoft.AspNetCore.Builder; +using Service.Storage.Storage; +using Service.Storage.Messaging; + +namespace Service.Storage; + +/// +/// Экстеншен для добавления брокера в зависимости от конфигурации приложения +/// +internal static class WebApplicationExtensions +{ + /// + /// Конфигурирует клиенские службы для взаимодействия с S3 + /// + /// Билдер + /// Билдер + public static async Task UseS3(this WebApplication app) + { + using var scope = app.Services.CreateScope(); + var s3Service = scope.ServiceProvider.GetRequiredService(); + await s3Service.EnsureBucketExists(); + return app; + } +} diff --git a/Service.Storage/appsettings.Development.json b/Service.Storage/appsettings.Development.json new file mode 100644 index 00000000..b2dcdb67 --- /dev/null +++ b/Service.Storage/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/Service.Storage/appsettings.json b/Service.Storage/appsettings.json new file mode 100644 index 00000000..3af72038 --- /dev/null +++ b/Service.Storage/appsettings.json @@ -0,0 +1,13 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "Settings": { + "MessageBroker": "", + "S3Hosting": "" + } +} \ No newline at end of file