From 7a0db391b0097364783570fef8e5515132181a23 Mon Sep 17 00:00:00 2001 From: Sergey Krylov Date: Fri, 27 Mar 2026 05:25:15 +0300 Subject: [PATCH] feat: add metrics --- compose.yml | 2 + package.json | 2 + src/app.module.ts | 2 + src/infra/rmq/rmq.service.ts | 25 +++++- src/main.ts | 2 +- .../notifications/notifications.controller.ts | 77 +++++++++++++++++-- src/observability/metrics/metrics.module.ts | 48 ++++++++++++ src/observability/observability.module.ts | 8 ++ yarn.lock | 30 ++++++++ 9 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 src/observability/metrics/metrics.module.ts create mode 100644 src/observability/observability.module.ts diff --git a/compose.yml b/compose.yml index 92fabdf..de5de6a 100644 --- a/compose.yml +++ b/compose.yml @@ -9,6 +9,8 @@ services: restart: always env_file: - .env.production.local + expose: + - '9102' networks: - teacinema diff --git a/package.json b/package.json index b3a3f97..75ac24b 100644 --- a/package.json +++ b/package.json @@ -30,11 +30,13 @@ "@nestjs/platform-express": "^11.0.1", "@teacinema/contracts": "^1.1.0", "@teacinema/core": "^1.0.9", + "@willsoto/nestjs-prometheus": "^6.0.2", "amqp-connection-manager": "^5.0.0", "amqplib": "^0.10.9", "axios": "^1.13.5", "cross-env": "^10.1.0", "handlebars": "^4.7.8", + "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", "zod": "^4.3.6" diff --git a/src/app.module.ts b/src/app.module.ts index b95dbc4..fbdc421 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -6,6 +6,7 @@ import { MailModule } from './infra/mail/mail.module' import { RmqModule } from './infra/rmq/rmq.module' import { SmsModule } from './infra/sms/sms.module' import { NotificationsModule } from './modules/notifications/notifications.module' +import { ObservabilityModule } from './observability/observability.module' @Module({ imports: [ @@ -20,6 +21,7 @@ import { NotificationsModule } from './modules/notifications/notifications.modul ] }), RmqModule, + ObservabilityModule, NotificationsModule, MailModule, SmsModule diff --git a/src/infra/rmq/rmq.service.ts b/src/infra/rmq/rmq.service.ts index 31c1bd0..c782598 100644 --- a/src/infra/rmq/rmq.service.ts +++ b/src/infra/rmq/rmq.service.ts @@ -1,11 +1,21 @@ import { Injectable, Logger } from '@nestjs/common' import { RmqContext } from '@nestjs/microservices' +import { InjectMetric } from '@willsoto/nestjs-prometheus' +import { Counter } from 'prom-client' @Injectable() export class RmqService { + private readonly SERVICE_NAME = 'notification-service' private logger = new Logger(RmqService.name) - public ack(context: RmqContext) { + public constructor( + @InjectMetric('rmq_events_ack_total') + private readonly ackTotal: Counter, + @InjectMetric('rmq_events_nack_total') + private readonly nackTotal: Counter + ) {} + + public ack(context: RmqContext, event: string) { const channel = context.getChannelRef() const msg = context.getMessage() const tag = msg?.fields?.deliveryTag @@ -14,11 +24,16 @@ export class RmqService { return } + this.ackTotal.inc({ + service: this.SERVICE_NAME, + event + }) + channel.ack(msg) this.logger.debug(`ACK (pattern: ${context.getPattern()}, tag: ${tag}): }`) } - public nack(context: RmqContext, requeue = false) { + public nack(context: RmqContext, event: string, requeue = false) { const channel = context.getChannelRef() const msg = context.getMessage() const tag = msg?.fields?.deliveryTag @@ -28,6 +43,12 @@ export class RmqService { } channel.nack(msg, false, requeue) + + this.nackTotal.inc({ + service: this.SERVICE_NAME, + event + }) + if (requeue) { this.logger.warn( `NACK response (pattern: ${context.getPattern()}, tag: ${tag}): }` diff --git a/src/main.ts b/src/main.ts index ef14e7a..973dcbc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -24,6 +24,6 @@ async function bootstrap() { }) await app.startAllMicroservices() - await app.init() + await app.listen(9102) } bootstrap() diff --git a/src/modules/notifications/notifications.controller.ts b/src/modules/notifications/notifications.controller.ts index d933d2d..a6a7b2c 100644 --- a/src/modules/notifications/notifications.controller.ts +++ b/src/modules/notifications/notifications.controller.ts @@ -5,6 +5,8 @@ import type { OtpRequestedEvent, PhoneChangedEvent } from '@teacinema/contracts' +import { InjectMetric } from '@willsoto/nestjs-prometheus' +import { Counter, Histogram } from 'prom-client' import { RmqService } from '../../infra/rmq/rmq.service' @@ -12,11 +14,16 @@ import { NotificationsService } from './notifications.service' @Controller() export class NotificationsController { + private readonly SERVICE_NAME = 'notification-service' private readonly logger = new Logger(NotificationsController.name) constructor( private readonly notificationsService: NotificationsService, - private readonly rmqService: RmqService + private readonly rmqService: RmqService, + @InjectMetric('rmq_event_processing_duration_seconds') + private readonly processingDuration: Histogram, + @InjectMetric('rmq_events_total') + private readonly eventsTotal: Counter ) {} @EventPattern('auth.otp.requested') @@ -24,12 +31,32 @@ export class NotificationsController { @Payload() data: OtpRequestedEvent, @Ctx() ctx: RmqContext ) { + const event = 'auth.otp.requested' + const endTimer = this.processingDuration.startTimer({ + service: this.SERVICE_NAME, + event + }) + try { await this.notificationsService.sendOtp(data) - this.rmqService.ack(ctx) + + this.eventsTotal.inc({ + service: this.SERVICE_NAME, + event, + status: 'success' + }) + + this.rmqService.ack(ctx, event) } catch (e) { + this.eventsTotal.inc({ + service: this.SERVICE_NAME, + event, + status: 'error' + }) this.logger.error(`OTP processing error ${e.message ?? e}}`) - this.rmqService.nack(ctx) + this.rmqService.nack(ctx, event) + } finally { + endTimer() } } @@ -38,12 +65,30 @@ export class NotificationsController { @Payload() data: PhoneChangedEvent, @Ctx() ctx: RmqContext ) { + const event = 'account.phone.changed' + const endTimer = this.processingDuration.startTimer({ + service: this.SERVICE_NAME, + event + }) + try { await this.notificationsService.phoneChanged(data) - this.rmqService.ack(ctx) + this.eventsTotal.inc({ + service: this.SERVICE_NAME, + event, + status: 'success' + }) + this.rmqService.ack(ctx, event) } catch (e) { + this.eventsTotal.inc({ + service: this.SERVICE_NAME, + event, + status: 'error' + }) this.logger.error(`Phone changing processing error ${e.message ?? e}}`) - this.rmqService.nack(ctx) + this.rmqService.nack(ctx, event) + } finally { + endTimer() } } @@ -52,12 +97,30 @@ export class NotificationsController { @Payload() data: EmailChangedEvent, @Ctx() ctx: RmqContext ) { + const event = 'account.email.changed' + const endTimer = this.processingDuration.startTimer({ + service: this.SERVICE_NAME, + event + }) + try { await this.notificationsService.emailChanged(data) - this.rmqService.ack(ctx) + this.eventsTotal.inc({ + service: this.SERVICE_NAME, + event, + status: 'success' + }) + this.rmqService.ack(ctx, event) } catch (e) { + this.eventsTotal.inc({ + service: this.SERVICE_NAME, + event, + status: 'error' + }) this.logger.error(`Email changing processing error ${e.message ?? e}}`) - this.rmqService.nack(ctx) + this.rmqService.nack(ctx, event) + } finally { + endTimer() } } } diff --git a/src/observability/metrics/metrics.module.ts b/src/observability/metrics/metrics.module.ts new file mode 100644 index 0000000..6c42cc9 --- /dev/null +++ b/src/observability/metrics/metrics.module.ts @@ -0,0 +1,48 @@ +import { Global, Module } from '@nestjs/common' +import { + makeCounterProvider, + makeHistogramProvider, + PrometheusModule +} from '@willsoto/nestjs-prometheus' + +@Global() +@Module({ + imports: [ + PrometheusModule.register({ + path: '/metrics', + defaultMetrics: { + enabled: true + } + }) + ], + providers: [ + makeHistogramProvider({ + name: 'rmq_event_processing_duration_seconds', + help: 'RabbitMQ event processing duration', + labelNames: ['service', 'event'], + buckets: [0.05, 0.1, 0.2, 0.5, 1, 2, 5] + }), + makeCounterProvider({ + name: 'rmq_events_total', + help: 'Total RMQ events processed', + labelNames: ['service', 'event', 'status'] + }), + makeCounterProvider({ + name: 'rmq_events_ack_total', + help: 'Total ACKed RMQ events', + labelNames: ['service', 'event'] + }), + makeCounterProvider({ + name: 'rmq_events_nack_total', + help: 'Total NACKed RMQ events', + labelNames: ['service', 'event'] + }) + ], + exports: [ + 'PROM_METRIC_RMQ_EVENT_PROCESSING_DURATION_SECONDS', + 'PROM_METRIC_RMQ_EVENTS_TOTAL', + 'PROM_METRIC_RMQ_EVENTS_ACK_TOTAL', + 'PROM_METRIC_RMQ_EVENTS_NACK_TOTAL' + ] +}) +export class MetricsModule {} diff --git a/src/observability/observability.module.ts b/src/observability/observability.module.ts new file mode 100644 index 0000000..857e458 --- /dev/null +++ b/src/observability/observability.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common' + +import { MetricsModule } from './metrics/metrics.module' + +@Module({ + imports: [MetricsModule] +}) +export class ObservabilityModule {} diff --git a/yarn.lock b/yarn.lock index 949e3ff..556a74b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1150,6 +1150,11 @@ resolved "https://registry.yarnpkg.com/@one-ini/wasm/-/wasm-0.1.1.tgz#6013659736c9dbfccc96e8a9c2b3de317df39323" integrity sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw== +"@opentelemetry/api@^1.4.0": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@paralleldrive/cuid2@^2.2.2": version "2.3.1" resolved "https://registry.yarnpkg.com/@paralleldrive/cuid2/-/cuid2-2.3.1.tgz#3d62ea9e7be867d3fa94b9897fab5b0ae187d784" @@ -1813,6 +1818,11 @@ "@webassemblyjs/ast" "1.14.1" "@xtuc/long" "4.2.2" +"@willsoto/nestjs-prometheus@^6.0.2": + version "6.0.2" + resolved "https://registry.yarnpkg.com/@willsoto/nestjs-prometheus/-/nestjs-prometheus-6.0.2.tgz#d603764a923848442ed092411716c0bf211de01f" + integrity sha512-ePyLZYdIrOOdlOWovzzMisIgviXqhPVzFpSMKNNhn6xajhRHeBsjAzSdpxZTc6pnjR9hw1lNAHyKnKl7lAPaVg== + "@xtuc/ieee754@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" @@ -2151,6 +2161,11 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== +bintrees@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" + integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw== + bl@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" @@ -5509,6 +5524,14 @@ preview-email@^3.0.19: pug "^3.0.3" uuid "^9.0.1" +prom-client@^15.1.3: + version "15.1.3" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2" + integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g== + dependencies: + "@opentelemetry/api" "^1.4.0" + tdigest "^0.1.1" + promise-breaker@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/promise-breaker/-/promise-breaker-6.0.0.tgz#107d2b70f161236abdb4ac5a736c7eb8df489d0f" @@ -6210,6 +6233,13 @@ tapable@^2.2.1, tapable@^2.3.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.3.0.tgz#7e3ea6d5ca31ba8e078b560f0d83ce9a14aa8be6" integrity sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg== +tdigest@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced" + integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA== + dependencies: + bintrees "1.0.2" + terser-webpack-plugin@^5.3.16: version "5.3.16" resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz#741e448cc3f93d8026ebe4f7ef9e4afacfd56330"