diff --git a/src/app.module.ts b/src/app.module.ts index 9f70df804..03c6573f7 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -32,7 +32,7 @@ import { EventEmitterModule } from "@nestjs/event-emitter"; import { AdminModule } from "./admin/admin.module"; import { HealthModule } from "./health/health.module"; import { LoggerModule } from "./loggers/logger.module"; -import { MetricsAndLogsModule } from "./metrics-and-logs/metrics-and-logs.module"; +import { MetricsAndLogsModule } from "./metrics/metrics-and-logs.module"; @Module({ imports: [ diff --git a/src/metrics-and-logs/access-logs/access-logs.controller.ts b/src/metrics-and-logs/access-logs/access-logs.controller.ts deleted file mode 100644 index 0dabb0f5e..000000000 --- a/src/metrics-and-logs/access-logs/access-logs.controller.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Controller, Get, Query } from "@nestjs/common"; -import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger"; -import { AccessLogsService } from "./access-logs.service"; -@ApiTags("access-logs") -@Controller("access-logs") -export class AccessLogsController { - constructor(private readonly accessLogsService: AccessLogsService) {} - - //TODO: use correct checkpolies - // @UseGuards(PoliciesGuard) - @Get("/find") - @ApiOperation({ summary: "Get access logs" }) - @ApiQuery({ - name: "query", - description: `{ "endpoint": { "$regex": "20.500.12269", "$options": "i" } } \n -{ "userId": "example" } \n -{ "$or": [ { "userId": "example" }, { "endpoint": { "$regex": "20.500.12269", "$options": "i" } } ] }`, - required: false, - type: String, - }) - @ApiQuery({ - name: "projection", - description: '{"userId": 1, "endpoint": 1, "createdAt": 1}', - required: false, - type: String, - }) - @ApiQuery({ - name: "options", - description: - "{ limit: 10, skip: 5, sort: { createdAt: -1 }, maxTimeMS: 1000, hint: { createdAt: 1 } }", - required: false, - type: String, - }) - async find( - @Query("query") query: string, - @Query("projection") - projection?: string, - @Query("options") options?: string, - ) { - const parsedQuery = query ? JSON.parse(query) : {}; - const parsedOption = options ? JSON.parse(options) : {}; - const parsedProjection = projection ? JSON.parse(projection) : null; - return this.accessLogsService.find( - parsedQuery, - parsedProjection, - parsedOption, - ); - } -} diff --git a/src/metrics-and-logs/access-logs/access-logs.module.ts b/src/metrics-and-logs/access-logs/access-logs.module.ts deleted file mode 100644 index abd383adc..000000000 --- a/src/metrics-and-logs/access-logs/access-logs.module.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Module } from "@nestjs/common"; -import { AccessLogsService } from "./access-logs.service"; -import { MongooseModule } from "@nestjs/mongoose"; -import { AccessLog, AccessLogSchema } from "./schemas/access-log.schema"; -import { AccessLogsController } from "./access-logs.controller"; -import { CaslModule } from "src/casl/casl.module"; - -@Module({ - imports: [ - MongooseModule.forFeature([ - { - name: AccessLog.name, - schema: AccessLogSchema, - }, - ]), - CaslModule, - ], - providers: [AccessLogsService, AccessLogsController], - exports: [AccessLogsService, AccessLogsController], - controllers: [AccessLogsController], -}) -export class AccessLogsModule {} diff --git a/src/metrics-and-logs/access-logs/access-logs.service.ts b/src/metrics-and-logs/access-logs/access-logs.service.ts deleted file mode 100644 index a850992b2..000000000 --- a/src/metrics-and-logs/access-logs/access-logs.service.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { InjectModel } from "@nestjs/mongoose"; -import { Model, FilterQuery, QueryOptions, ProjectionType } from "mongoose"; -import { AccessLog, AccessLogDocument } from "./schemas/access-log.schema"; - -export class AccessLogsService { - constructor( - @InjectModel(AccessLog.name) - private accessLogModel: Model, - ) {} - - async create(accessLogData: AccessLog): Promise { - const accessLog = new this.accessLogModel(accessLogData); - return accessLog.save(); - } - - async find( - query: FilterQuery, - projection: ProjectionType | null | undefined, - options: QueryOptions | null | undefined, - ): Promise { - return this.accessLogModel.find(query, projection, options).exec(); - } -} diff --git a/src/metrics-and-logs/access-logs/interfaces/accessLogs.interface.ts b/src/metrics-and-logs/access-logs/interfaces/accessLogs.interface.ts deleted file mode 100644 index 101f928e9..000000000 --- a/src/metrics-and-logs/access-logs/interfaces/accessLogs.interface.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface IAccessLogs { - user: string | null; - ip: string | undefined; - userAgent: string | undefined; - endpoint: string; - statusCode: number; - responseTime: number; -} diff --git a/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts b/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts deleted file mode 100644 index 523d87e95..000000000 --- a/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; -import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; -import { Document } from "mongoose"; - -export type AccessLogDocument = AccessLog & Document; - -@Schema({ - collection: "AccessLog", - timestamps: { createdAt: true, updatedAt: false }, - versionKey: false, -}) -export class AccessLog { - @ApiPropertyOptional({ - description: "User ID associated with the access log", - type: String, - }) - @Prop({ type: String, default: null }) - userId: string | null; - - @ApiPropertyOptional({ - description: "The timestamp when the document was created", - type: Date, - }) - @Prop({ type: Date, default: Date.now }) - createdAt?: Date; - - @ApiProperty({ - description: "HTTP status code for the request", - type: Number, - }) - @Prop({ type: Number, required: true }) - statusCode: number; - - @ApiProperty({ - description: "Endpoint for the request", - type: String, - }) - @Prop({ type: String, required: true }) - endpoint: string; - - @ApiPropertyOptional({ - description: "Endpoint for the request", - type: Object, - }) - @Prop({ type: Object, default: null }) - query: object; - - @ApiPropertyOptional({ - description: "Original IP of the user for the request", - type: String, - }) - @Prop({ type: String, default: "Not found" }) - originIp: string | undefined; - - @ApiProperty({ - description: "Response time in ms for the request", - type: String, - }) - @Prop({ type: Number, required: true }) - responseTime: number; -} - -export const AccessLogSchema = SchemaFactory.createForClass(AccessLog); - -AccessLogSchema.index({ createdAt: 1 }); diff --git a/src/metrics-and-logs/metrics/interfaces/metrics.interface.ts b/src/metrics-and-logs/metrics/interfaces/metrics.interface.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/metrics-and-logs/metrics/metrics.controller.ts b/src/metrics-and-logs/metrics/metrics.controller.ts deleted file mode 100644 index eaf416f4d..000000000 --- a/src/metrics-and-logs/metrics/metrics.controller.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Controller, Get, Query } from "@nestjs/common"; -import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger"; -import { MetricsService } from "./metrics.service"; - -@ApiTags("metrics") -@Controller("metrics") -export class MetricsController { - constructor(private readonly metricsService: MetricsService) {} - - //TODO: correct checkpolies and api description - @Get("/find") - @ApiOperation({ summary: "Get metrics" }) - @ApiQuery({ - name: "query", - description: `{ "endpoint": { "$regex": "20.500.12269", "$options": "i" } } \n -{ "userId": "example" } \n -{ "$or": [ { "userId": "example" }, { "endpoint": { "$regex": "20.500.12269", "$options": "i" } } ] }`, - required: false, - type: String, - }) - @ApiQuery({ - name: "projection", - description: '{"userId": 1, "endpoint": 1, "createdAt": 1}', - required: false, - type: String, - }) - @ApiQuery({ - name: "options", - description: - "{ limit: 10, skip: 5, sort: { createdAt: -1 }, maxTimeMS: 1000, hint: { createdAt: 1 } }", - required: false, - type: String, - }) - async find( - @Query("query") query: string, - @Query("projection") - projection?: string, - @Query("options") options?: string, - ) { - const parsedQuery = query ? JSON.parse(query) : {}; - const parsedOption = options ? JSON.parse(options) : {}; - const parsedProjection = projection ? JSON.parse(projection) : null; - return this.metricsService.find( - parsedQuery, - parsedProjection, - parsedOption, - ); - } -} diff --git a/src/metrics-and-logs/metrics/metrics.module.ts b/src/metrics-and-logs/metrics/metrics.module.ts deleted file mode 100644 index 308bc167b..000000000 --- a/src/metrics-and-logs/metrics/metrics.module.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Module } from "@nestjs/common"; -import { MetricsService } from "./metrics.service"; -import { MetricsController } from "./metrics.controller"; -import { MongooseModule } from "@nestjs/mongoose"; -import { Metrics, MetricsSchema } from "./schemas/metrics.schema"; -import { AccessLogsModule } from "../access-logs/access-logs.module"; - -@Module({ - imports: [ - MongooseModule.forFeature([ - { - name: Metrics.name, - schema: MetricsSchema, - }, - ]), - AccessLogsModule, - ], - providers: [MetricsService, MetricsController], - exports: [], -}) -export class MetricsModule {} diff --git a/src/metrics-and-logs/metrics/metrics.service.ts b/src/metrics-and-logs/metrics/metrics.service.ts deleted file mode 100644 index 10d9ae444..000000000 --- a/src/metrics-and-logs/metrics/metrics.service.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { InjectModel } from "@nestjs/mongoose"; -import { Metrics, MetricsDocument } from "./schemas/metrics.schema"; -import { FilterQuery, Model, ProjectionType, QueryOptions } from "mongoose"; -import { AccessLogsService } from "../access-logs/access-logs.service"; - -export class MetricsService { - constructor( - @InjectModel(Metrics.name) - private metricsModel: Model, - private readonly accessLogsService: AccessLogsService, - ) {} - async create(metricsData: Metrics): Promise { - const metrics = new this.metricsModel(metricsData); - return metrics.save(); - } - - async find( - query: FilterQuery, - projection: ProjectionType | null | undefined, - options: QueryOptions | null | undefined, - ): Promise { - return this.metricsModel.find(query, projection, options).exec(); - } -} diff --git a/src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts b/src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts deleted file mode 100644 index cea9b46b3..000000000 --- a/src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; -import { ApiProperty } from "@nestjs/swagger"; -import { Document } from "mongoose"; - -@Schema({ - versionKey: false, -}) -export class MetricsRecord { - @ApiProperty({ - description: "The endpoint path being tracked", - type: String, - }) - @Prop({ type: String, required: true }) - endpoint: string; - - @ApiProperty({ - description: "The records of individual access logs for this endpoint", - type: [Object], - }) - @Prop({ - type: [ - { - userId: { type: String, required: false }, - statusCode: { type: Number, required: true }, - date: { type: Date, required: true }, - }, - ], - required: true, - }) - records: Array<{ - userId: string | null; - statusCode: number; - date: Date; - }>; - - @ApiProperty({ - description: "Summary statistics of access for this endpoint", - type: Object, - }) - @Prop({ type: Object, required: true }) - summary: { - totalRequests: number; - }; -} - -// Create the schema for MetricsRecord -export const MetricsRecordSchema = SchemaFactory.createForClass(MetricsRecord); - -// If needed, export the type -export type MetricsRecordDocument = MetricsRecord & Document; diff --git a/src/metrics-and-logs/metrics/schemas/metrics.schema.ts b/src/metrics-and-logs/metrics/schemas/metrics.schema.ts deleted file mode 100644 index 630669a21..000000000 --- a/src/metrics-and-logs/metrics/schemas/metrics.schema.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; -import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; -import { Date, Document } from "mongoose"; -import { MetricsRecord } from "./metrics-record.schema"; - -export type MetricsDocument = Metrics & Document; - -@Schema({ - collection: "Metrics", - timestamps: { createdAt: true, updatedAt: false }, - versionKey: false, -}) -export class Metrics { - @ApiPropertyOptional({ - description: "The date the metric was recorded", - type: Date, - }) - @Prop({ type: Date, default: Date.now }) - createdAt?: Date; - - @ApiProperty({ - description: "A list of endpoints with their access log compacted details", - type: [Object], - }) - @Prop({ type: [MetricsRecord] }) - endpointMetrics: MetricsRecord[]; -} - -export const MetricsSchema = SchemaFactory.createForClass(Metrics); diff --git a/src/metrics-and-logs/metrics/tasks/metrics-cron.task.ts b/src/metrics-and-logs/metrics/tasks/metrics-cron.task.ts deleted file mode 100644 index 127ec0607..000000000 --- a/src/metrics-and-logs/metrics/tasks/metrics-cron.task.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Injectable, Logger } from "@nestjs/common"; -import { Cron } from "@nestjs/schedule"; -import { MetricsController } from "../metrics.controller"; - -@Injectable() -export class MetricsCronTask { - constructor(private readonly metricsController: MetricsController) {} - - @Cron("0 0 * * *") // Run daily at midnight - async handleCron() { - Logger.log("Running daily metrics compaction..."); - } -} diff --git a/src/metrics-and-logs/metrics-and-logs.module.ts b/src/metrics/metrics-and-logs.module.ts similarity index 77% rename from src/metrics-and-logs/metrics-and-logs.module.ts rename to src/metrics/metrics-and-logs.module.ts index 1e97d3e16..075b7246a 100644 --- a/src/metrics-and-logs/metrics-and-logs.module.ts +++ b/src/metrics/metrics-and-logs.module.ts @@ -1,13 +1,11 @@ import { Logger, MiddlewareConsumer, Module, NestModule } from "@nestjs/common"; -import { MetricsModule } from "./metrics/metrics.module"; -import { AccessLogsModule } from "./access-logs/access-logs.module"; import { ConfigModule, ConfigService } from "@nestjs/config"; import { MetricsAndLogsMiddleware } from "./middlewares/metrics-and-logs.middleware"; import { JwtModule } from "@nestjs/jwt"; @Module({ - imports: [MetricsModule, AccessLogsModule, ConfigModule, JwtModule], - exports: [MetricsModule, AccessLogsModule], + imports: [ConfigModule, JwtModule], + exports: [], }) export class MetricsAndLogsModule implements NestModule { constructor(private readonly configService: ConfigService) {} diff --git a/src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts b/src/metrics/middlewares/metrics-and-logs.middleware.ts similarity index 75% rename from src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts rename to src/metrics/middlewares/metrics-and-logs.middleware.ts index 97d8b1c89..e82348ac9 100644 --- a/src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts +++ b/src/metrics/middlewares/metrics-and-logs.middleware.ts @@ -1,20 +1,20 @@ import { Injectable, Logger, NestMiddleware } from "@nestjs/common"; import { Request, Response, NextFunction } from "express"; import { JwtService } from "@nestjs/jwt"; -import { AccessLogsService } from "../access-logs/access-logs.service"; import { parse } from "url"; @Injectable() export class MetricsAndLogsMiddleware implements NestMiddleware { private requestCache = new Map(); // Cache to store recent requests - private cacheDuration = 1000; + private cacheStoreDuration = 1000; + private cacheResetInterval = 10 * 60 * 1000; // 10 minutes - constructor( - private readonly jwtService: JwtService, - private readonly accessLogsService: AccessLogsService, - ) {} + constructor(private readonly jwtService: JwtService) { + this.startCacheResetInterval(); + } use(req: Request, res: Response, next: NextFunction) { const { query, pathname } = parse(req.originalUrl, true); + const userAgent = req.headers["user-agent"]; // TODO: Better to use a library for this? const isBot = userAgent ? /bot|crawl|spider|slurp/i.test(userAgent) : false; @@ -31,13 +31,16 @@ export class MetricsAndLogsMiddleware implements NestMiddleware { res.on("finish", () => { const statusCode = res.statusCode; + if (statusCode === 304) return; + const responseTime = Date.now() - startTime; const lastHitTime = this.requestCache.get(cacheKeyIdentifier); + console.log("===qieru", query); // Log only if the request was not recently logged - if (!lastHitTime || Date.now() - lastHitTime > this.cacheDuration) { - this.accessLogsService.create({ + if (!lastHitTime || Date.now() - lastHitTime > this.cacheStoreDuration) { + Logger.log("SciCatAccessLogs", { userId, originIp, endpoint: pathname, @@ -54,9 +57,9 @@ export class MetricsAndLogsMiddleware implements NestMiddleware { } private parseToken(authHeader?: string) { - if (!authHeader) return null; + if (!authHeader) return "anonymous"; const token = authHeader.split(" ")[1]; - if (!token) return null; + if (!token) return "anonymous"; try { const { id } = this.jwtService.decode(token); @@ -66,4 +69,10 @@ export class MetricsAndLogsMiddleware implements NestMiddleware { return null; } } + + private startCacheResetInterval() { + setInterval(() => { + this.requestCache.clear(); + }, this.cacheResetInterval); + } }