diff --git a/.github/workflows/api-cd.yml b/.github/workflows/api-cd.yml index bcbcf26..8a85820 100644 --- a/.github/workflows/api-cd.yml +++ b/.github/workflows/api-cd.yml @@ -64,10 +64,17 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + migrate: + name: DB Migrate + needs: ci + uses: ./.github/workflows/db-migrate.yml + secrets: + SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} + deploy: name: Deploy to API Server runs-on: ubuntu-latest - needs: build-and-push + needs: [build-and-push, migrate] steps: - name: Checkout Repository diff --git a/.github/workflows/batch-cd.yml b/.github/workflows/batch-cd.yml index bf42384..bb3fcd7 100644 --- a/.github/workflows/batch-cd.yml +++ b/.github/workflows/batch-cd.yml @@ -64,10 +64,17 @@ jobs: cache-from: type=gha,scope=batch cache-to: type=gha,mode=max,scope=batch + migrate: + name: DB Migrate + needs: ci + uses: ./.github/workflows/db-migrate.yml + secrets: + SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} + deploy: name: Deploy to API Server runs-on: ubuntu-latest - needs: build-and-push + needs: [build-and-push, migrate] steps: - name: Checkout Repository diff --git a/.github/workflows/db-migrate.yml b/.github/workflows/db-migrate.yml new file mode 100644 index 0000000..733ac7c --- /dev/null +++ b/.github/workflows/db-migrate.yml @@ -0,0 +1,41 @@ +name: DB Migrate + +# API/Batch CD에서 배포 전에 호출한다. 마이그레이션만 바꾼 경우에는 수동 실행(workflow_dispatch)한다. +on: + workflow_call: + secrets: + SUPABASE_DB_URL: + required: true + + workflow_dispatch: + +permissions: + contents: read + +jobs: + migrate: + name: Apply Supabase Migrations + runs-on: ubuntu-latest + # API/Batch CD가 동시에 호출해도 순서대로 적용한다 (이미 적용된 마이그레이션은 건너뜀) + concurrency: + group: db-migrate + cancel-in-progress: false + + steps: + - name: Checkout Repository + uses: actions/checkout@v7 + + - name: Set up Supabase CLI + uses: supabase/setup-cli@v1 + with: + version: latest + + - name: Show Migration Status + run: supabase migration list --db-url "$SUPABASE_DB_URL" + env: + SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} + + - name: Apply Migrations + run: supabase db push --db-url "$SUPABASE_DB_URL" + env: + SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} diff --git a/services/api/app/src/api/admin/common/guards/admin-api-key.guard.spec.ts b/services/api/app/src/api/admin/common/guards/admin-api-key.guard.spec.ts new file mode 100644 index 0000000..19938d3 --- /dev/null +++ b/services/api/app/src/api/admin/common/guards/admin-api-key.guard.spec.ts @@ -0,0 +1,34 @@ +import { ExecutionContext } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { AdminApiKeyGuard } from 'src/api/admin/common/guards/admin-api-key.guard'; + +function createContext(headers: Record): ExecutionContext { + return { + switchToHttp: () => ({ getRequest: () => ({ headers }) }), + } as unknown as ExecutionContext; +} + +function createGuard(apiKey: string | undefined): AdminApiKeyGuard { + return new AdminApiKeyGuard({ get: () => apiKey } as unknown as ConfigService); +} + +describe('AdminApiKeyGuard', () => { + it('헤더의 키가 일치하면 통과시킨다', () => { + const guard = createGuard('secret'); + + expect(guard.canActivate(createContext({ 'x-admin-api-key': 'secret' }))).toBe(true); + }); + + it('헤더의 키가 다르거나 없으면 막는다', () => { + const guard = createGuard('secret'); + + expect(guard.canActivate(createContext({ 'x-admin-api-key': 'wrong' }))).toBe(false); + expect(guard.canActivate(createContext({}))).toBe(false); + }); + + it('ADMIN_API_KEY가 설정되지 않았으면 항상 막는다', () => { + const guard = createGuard(undefined); + + expect(guard.canActivate(createContext({ 'x-admin-api-key': '' }))).toBe(false); + }); +}); diff --git a/services/api/app/src/api/admin/common/guards/admin-api-key.guard.ts b/services/api/app/src/api/admin/common/guards/admin-api-key.guard.ts new file mode 100644 index 0000000..a35e513 --- /dev/null +++ b/services/api/app/src/api/admin/common/guards/admin-api-key.guard.ts @@ -0,0 +1,26 @@ +import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { createHash, timingSafeEqual } from 'crypto'; + +export const ADMIN_API_KEY_HEADER = 'x-admin-api-key'; + +@Injectable() +export class AdminApiKeyGuard implements CanActivate { + constructor(private readonly configService: ConfigService) {} + + canActivate(context: ExecutionContext): boolean { + const expected = this.configService.get('ADMIN_API_KEY'); + // 키가 설정되지 않은 환경에서는 admin API를 막는다 + if (!expected) return false; + + const request = context.switchToHttp().getRequest(); + const provided = request.headers[ADMIN_API_KEY_HEADER]; + if (typeof provided !== 'string') return false; + + return timingSafeEqual(digest(provided), digest(expected)); + } +} + +function digest(value: string): Buffer { + return createHash('sha256').update(value).digest(); +} diff --git a/services/api/app/src/api/admin/scrape-runs/application/dtos/results/scraper-status-result.dto.ts b/services/api/app/src/api/admin/scrape-runs/application/dtos/results/scraper-status-result.dto.ts new file mode 100644 index 0000000..f7691d3 --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/application/dtos/results/scraper-status-result.dto.ts @@ -0,0 +1,15 @@ +import { + ScrapeRun, + ScrapeRunType, +} from 'src/api/admin/scrape-runs/domain/entities/scrape-run.entity'; + +export interface ScraperStatusResult { + type: ScrapeRunType; + latestRun: ScrapeRun | null; + lastSucceededRun: ScrapeRun | null; +} + +export interface ScrapeRunListResult { + runs: ScrapeRun[]; + nextCursor: number | null; +} diff --git a/services/api/app/src/api/admin/scrape-runs/application/scrape-runs.service.spec.ts b/services/api/app/src/api/admin/scrape-runs/application/scrape-runs.service.spec.ts new file mode 100644 index 0000000..a667a23 --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/application/scrape-runs.service.spec.ts @@ -0,0 +1,100 @@ +import { ConflictException, NotFoundException } from '@nestjs/common'; +import { ScrapeRunsService } from 'src/api/admin/scrape-runs/application/scrape-runs.service'; +import { ScrapeRun } from 'src/api/admin/scrape-runs/domain/entities/scrape-run.entity'; +import { ScrapeRunRepository } from 'src/api/admin/scrape-runs/infrastructure/scrape-run.repository'; + +function createRun(overrides: Partial): ScrapeRun { + return { + id: 1, + type: 'shuttle', + trigger: 'cron', + status: 'succeeded', + errorMessage: null, + createdAt: new Date('2026-09-25T00:00:00.000Z'), + startedAt: null, + finishedAt: null, + ...overrides, + }; +} + +describe('ScrapeRunsService', () => { + let repository: jest.Mocked; + let service: ScrapeRunsService; + + beforeEach(() => { + repository = { + findMany: jest.fn(), + findById: jest.fn(), + findLatestPerType: jest.fn(), + findLastSucceededPerType: jest.fn(), + createPending: jest.fn(), + } as unknown as jest.Mocked; + service = new ScrapeRunsService(repository); + }); + + describe('getRuns', () => { + it('limit보다 많이 조회되면 마지막 항목 id를 다음 커서로 반환한다', async () => { + repository.findMany.mockResolvedValue([ + createRun({ id: 5 }), + createRun({ id: 4 }), + createRun({ id: 3 }), + ]); + + const result = await service.getRuns({ type: 'shuttle', limit: 2 }); + + expect(repository.findMany).toHaveBeenCalledWith({ type: 'shuttle', limit: 3 }); + expect(result.runs.map(run => run.id)).toEqual([5, 4]); + expect(result.nextCursor).toBe(4); + }); + + it('마지막 페이지면 다음 커서가 null이다', async () => { + repository.findMany.mockResolvedValue([createRun({ id: 1 })]); + + const result = await service.getRuns({ limit: 2 }); + + expect(result.nextCursor).toBeNull(); + }); + }); + + describe('getRun', () => { + it('run이 없으면 NotFoundException을 던진다', async () => { + repository.findById.mockResolvedValue(null); + + await expect(service.getRun(1)).rejects.toThrow(NotFoundException); + }); + }); + + describe('requestRun', () => { + it('대기 상태의 수동 run을 등록한다', async () => { + const run = createRun({ id: 7, trigger: 'manual', status: 'pending' }); + repository.createPending.mockResolvedValue(run); + + await expect(service.requestRun('shuttle')).resolves.toBe(run); + expect(repository.createPending).toHaveBeenCalledWith('shuttle'); + }); + + it('같은 타입이 이미 대기/실행 중이면 ConflictException을 던진다', async () => { + repository.createPending.mockResolvedValue(null); + + await expect(service.requestRun('shuttle')).rejects.toThrow(ConflictException); + }); + }); + + describe('getScraperStatuses', () => { + it('모든 수집 타입에 대해 최근 run과 최근 성공 run을 반환한다', async () => { + const failedShuttle = createRun({ id: 3, status: 'failed' }); + const succeededShuttle = createRun({ id: 2 }); + repository.findLatestPerType.mockResolvedValue([failedShuttle]); + repository.findLastSucceededPerType.mockResolvedValue([succeededShuttle]); + + const result = await service.getScraperStatuses(); + + expect(result).toEqual([ + { type: 'shuttle', latestRun: failedShuttle, lastSucceededRun: succeededShuttle }, + { type: 'notice', latestRun: null, lastSucceededRun: null }, + { type: 'cafeteria', latestRun: null, lastSucceededRun: null }, + { type: 'academic-calendar', latestRun: null, lastSucceededRun: null }, + ]); + }); + }); +}); diff --git a/services/api/app/src/api/admin/scrape-runs/application/scrape-runs.service.ts b/services/api/app/src/api/admin/scrape-runs/application/scrape-runs.service.ts new file mode 100644 index 0000000..97bbc54 --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/application/scrape-runs.service.ts @@ -0,0 +1,63 @@ +import { ConflictException, Injectable, NotFoundException } from '@nestjs/common'; +import { + ScrapeRunListResult, + ScraperStatusResult, +} from 'src/api/admin/scrape-runs/application/dtos/results/scraper-status-result.dto'; +import { + SCRAPE_RUN_TYPES, + ScrapeRun, + ScrapeRunType, +} from 'src/api/admin/scrape-runs/domain/entities/scrape-run.entity'; +import { + ScrapeRunRepository, + ScrapeRunSearchCondition, +} from 'src/api/admin/scrape-runs/infrastructure/scrape-run.repository'; + +@Injectable() +export class ScrapeRunsService { + constructor(private readonly scrapeRunRepository: ScrapeRunRepository) {} + + public async getRuns(condition: ScrapeRunSearchCondition): Promise { + // 다음 페이지 존재 여부 확인을 위해 하나 더 조회한다 + const runs = await this.scrapeRunRepository.findMany({ + ...condition, + limit: condition.limit + 1, + }); + const hasNext = runs.length > condition.limit; + const page = hasNext ? runs.slice(0, condition.limit) : runs; + + return { + runs: page, + nextCursor: hasNext ? Number(page[page.length - 1].id) : null, + }; + } + + public async getRun(id: number): Promise { + const run = await this.scrapeRunRepository.findById(id); + if (!run) { + throw new NotFoundException(`${id}번 수집 실행 기록을 찾을 수 없습니다.`); + } + return run; + } + + public async requestRun(type: ScrapeRunType): Promise { + const run = await this.scrapeRunRepository.createPending(type); + if (!run) { + throw new ConflictException(`'${type}' 수집이 이미 대기 또는 실행 중입니다.`); + } + return run; + } + + public async getScraperStatuses(): Promise { + const [latestRuns, succeededRuns] = await Promise.all([ + this.scrapeRunRepository.findLatestPerType(), + this.scrapeRunRepository.findLastSucceededPerType(), + ]); + + return SCRAPE_RUN_TYPES.map(type => ({ + type, + latestRun: latestRuns.find(run => run.type === type) ?? null, + lastSucceededRun: succeededRuns.find(run => run.type === type) ?? null, + })); + } +} diff --git a/services/api/app/src/api/admin/scrape-runs/domain/entities/scrape-run.entity.ts b/services/api/app/src/api/admin/scrape-runs/domain/entities/scrape-run.entity.ts new file mode 100644 index 0000000..ece4e5f --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/domain/entities/scrape-run.entity.ts @@ -0,0 +1,36 @@ +import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; + +export const SCRAPE_RUN_TYPES = ['shuttle', 'notice', 'cafeteria', 'academic-calendar'] as const; +export type ScrapeRunType = (typeof SCRAPE_RUN_TYPES)[number]; + +export const SCRAPE_RUN_STATUSES = ['pending', 'running', 'succeeded', 'failed'] as const; +export type ScrapeRunStatus = (typeof SCRAPE_RUN_STATUSES)[number]; + +export type ScrapeRunTrigger = 'cron' | 'manual'; + +@Entity('scrape_run') +export class ScrapeRun { + @PrimaryGeneratedColumn() + id: number; + + @Column({ length: 30 }) + type: ScrapeRunType; + + @Column({ length: 10 }) + trigger: ScrapeRunTrigger; + + @Column({ length: 10 }) + status: ScrapeRunStatus; + + @Column({ name: 'error_message', type: 'text', nullable: true }) + errorMessage: string | null; + + @Column({ name: 'created_at', type: 'timestamptz' }) + createdAt: Date; + + @Column({ name: 'started_at', type: 'timestamptz', nullable: true }) + startedAt: Date | null; + + @Column({ name: 'finished_at', type: 'timestamptz', nullable: true }) + finishedAt: Date | null; +} diff --git a/services/api/app/src/api/admin/scrape-runs/infrastructure/scrape-run.repository.ts b/services/api/app/src/api/admin/scrape-runs/infrastructure/scrape-run.repository.ts new file mode 100644 index 0000000..e9df8dc --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/infrastructure/scrape-run.repository.ts @@ -0,0 +1,81 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { FindOptionsWhere, LessThan, QueryFailedError, Repository } from 'typeorm'; +import { + ScrapeRun, + ScrapeRunStatus, + ScrapeRunType, +} from 'src/api/admin/scrape-runs/domain/entities/scrape-run.entity'; + +const UNIQUE_VIOLATION = '23505'; + +export interface ScrapeRunSearchCondition { + type?: ScrapeRunType; + status?: ScrapeRunStatus; + cursor?: number; + limit: number; +} + +@Injectable() +export class ScrapeRunRepository { + constructor( + @InjectRepository(ScrapeRun) + private readonly scrapeRunRepository: Repository, + ) {} + + findMany({ type, status, cursor, limit }: ScrapeRunSearchCondition): Promise { + const where: FindOptionsWhere = {}; + if (type) where.type = type; + if (status) where.status = status; + if (cursor) where.id = LessThan(cursor); + + return this.scrapeRunRepository.find({ where, order: { id: 'DESC' }, take: limit }); + } + + findById(id: number): Promise { + return this.scrapeRunRepository.findOne({ where: { id } }); + } + + /** 타입별 가장 최근 run */ + findLatestPerType(): Promise { + return this.scrapeRunRepository + .createQueryBuilder('run') + .distinctOn(['run.type']) + .orderBy('run.type') + .addOrderBy('run.id', 'DESC') + .getMany(); + } + + /** 타입별 가장 최근 성공 run */ + findLastSucceededPerType(): Promise { + return this.scrapeRunRepository + .createQueryBuilder('run') + .distinctOn(['run.type']) + .where('run.status = :status', { status: 'succeeded' }) + .orderBy('run.type') + .addOrderBy('run.id', 'DESC') + .getMany(); + } + + /** + * 수동 실행 요청을 대기 상태로 등록한다. + * 같은 타입의 run이 이미 대기/실행 중이면 null을 반환한다. + */ + async createPending(type: ScrapeRunType): Promise { + try { + const run = this.scrapeRunRepository.create({ type, trigger: 'manual', status: 'pending' }); + const { id } = await this.scrapeRunRepository.save(run); + return this.findById(id); + } catch (error) { + if (isUniqueViolation(error)) return null; + throw error; + } + } +} + +function isUniqueViolation(error: unknown): boolean { + return ( + error instanceof QueryFailedError && + (error.driverError as { code?: string }).code === UNIQUE_VIOLATION + ); +} diff --git a/services/api/app/src/api/admin/scrape-runs/presentation/dtos/requests/create-scrape-run-request.dto.ts b/services/api/app/src/api/admin/scrape-runs/presentation/dtos/requests/create-scrape-run-request.dto.ts new file mode 100644 index 0000000..0d625fd --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/presentation/dtos/requests/create-scrape-run-request.dto.ts @@ -0,0 +1,12 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsIn } from 'class-validator'; +import { + SCRAPE_RUN_TYPES, + ScrapeRunType, +} from 'src/api/admin/scrape-runs/domain/entities/scrape-run.entity'; + +export class CreateScrapeRunRequestDto { + @IsIn(SCRAPE_RUN_TYPES) + @ApiProperty({ description: '수집 타입', enum: SCRAPE_RUN_TYPES, example: 'shuttle' }) + type: ScrapeRunType; +} diff --git a/services/api/app/src/api/admin/scrape-runs/presentation/dtos/requests/list-scrape-runs-query.dto.ts b/services/api/app/src/api/admin/scrape-runs/presentation/dtos/requests/list-scrape-runs-query.dto.ts new file mode 100644 index 0000000..c558e56 --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/presentation/dtos/requests/list-scrape-runs-query.dto.ts @@ -0,0 +1,36 @@ +import { ApiPropertyOptional } from '@nestjs/swagger'; +import { Type } from 'class-transformer'; +import { IsIn, IsInt, IsOptional, Max, Min } from 'class-validator'; +import { + SCRAPE_RUN_STATUSES, + SCRAPE_RUN_TYPES, + ScrapeRunStatus, + ScrapeRunType, +} from 'src/api/admin/scrape-runs/domain/entities/scrape-run.entity'; + +export class ListScrapeRunsQueryDto { + @IsOptional() + @IsIn(SCRAPE_RUN_TYPES) + @ApiPropertyOptional({ description: '수집 타입', enum: SCRAPE_RUN_TYPES }) + type?: ScrapeRunType; + + @IsOptional() + @IsIn(SCRAPE_RUN_STATUSES) + @ApiPropertyOptional({ description: '실행 상태', enum: SCRAPE_RUN_STATUSES }) + status?: ScrapeRunStatus; + + @IsOptional() + @IsInt() + @Min(1) + @Type(() => Number) + @ApiPropertyOptional({ description: '이전 페이지의 nextCursor. 이 id보다 오래된 run을 조회' }) + cursor?: number; + + @IsOptional() + @IsInt() + @Min(1) + @Max(100) + @Type(() => Number) + @ApiPropertyOptional({ description: '페이지 크기', default: 20, maximum: 100 }) + limit?: number; +} diff --git a/services/api/app/src/api/admin/scrape-runs/presentation/dtos/responses/scrape-run-response.dto.ts b/services/api/app/src/api/admin/scrape-runs/presentation/dtos/responses/scrape-run-response.dto.ts new file mode 100644 index 0000000..1214fd6 --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/presentation/dtos/responses/scrape-run-response.dto.ts @@ -0,0 +1,71 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { + SCRAPE_RUN_STATUSES, + SCRAPE_RUN_TYPES, + ScrapeRun, + ScrapeRunStatus, + ScrapeRunTrigger, + ScrapeRunType, +} from 'src/api/admin/scrape-runs/domain/entities/scrape-run.entity'; + +export class ScrapeRunResponseDto { + @ApiProperty({ example: 1 }) + id: number; + + @ApiProperty({ enum: SCRAPE_RUN_TYPES }) + type: ScrapeRunType; + + @ApiProperty({ enum: ['cron', 'manual'] }) + trigger: ScrapeRunTrigger; + + @ApiProperty({ enum: SCRAPE_RUN_STATUSES }) + status: ScrapeRunStatus; + + @ApiProperty({ nullable: true, type: String }) + errorMessage: string | null; + + @ApiProperty({ description: '요청/생성 일시 (ISO 8601)' }) + createdAt: string; + + @ApiProperty({ nullable: true, type: String }) + startedAt: string | null; + + @ApiProperty({ nullable: true, type: String }) + finishedAt: string | null; + + static from(run: ScrapeRun): ScrapeRunResponseDto { + return { + id: Number(run.id), + type: run.type, + trigger: run.trigger, + status: run.status, + errorMessage: run.errorMessage, + createdAt: run.createdAt.toISOString(), + startedAt: run.startedAt?.toISOString() ?? null, + finishedAt: run.finishedAt?.toISOString() ?? null, + }; + } +} + +export class ScrapeRunListResponseDto { + @ApiProperty({ type: [ScrapeRunResponseDto] }) + items: ScrapeRunResponseDto[]; + + @ApiProperty({ + nullable: true, + type: Number, + description: '다음 페이지 커서. 마지막 페이지면 null', + }) + nextCursor: number | null; +} + +export class ScraperStatusResponseDto { + @ApiProperty({ enum: SCRAPE_RUN_TYPES }) + type: ScrapeRunType; + + @ApiProperty({ nullable: true, type: ScrapeRunResponseDto }) + latestRun: ScrapeRunResponseDto | null; + + @ApiProperty({ nullable: true, type: ScrapeRunResponseDto }) + lastSucceededRun: ScrapeRunResponseDto | null; +} diff --git a/services/api/app/src/api/admin/scrape-runs/presentation/scrape-runs.controller.ts b/services/api/app/src/api/admin/scrape-runs/presentation/scrape-runs.controller.ts new file mode 100644 index 0000000..a63d8e8 --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/presentation/scrape-runs.controller.ts @@ -0,0 +1,85 @@ +import { + Body, + Controller, + Get, + HttpCode, + HttpStatus, + Param, + ParseIntPipe, + Post, + Query, + UseGuards, +} from '@nestjs/common'; +import { ApiAcceptedResponse, ApiOkResponse, ApiSecurity, ApiTags } from '@nestjs/swagger'; +import { AdminApiKeyGuard } from 'src/api/admin/common/guards/admin-api-key.guard'; +import { ScrapeRunsService } from 'src/api/admin/scrape-runs/application/scrape-runs.service'; +import { NativeResponseDto } from 'src/api/common/dtos/native-response.dto'; +import { CreateScrapeRunRequestDto } from './dtos/requests/create-scrape-run-request.dto'; +import { ListScrapeRunsQueryDto } from './dtos/requests/list-scrape-runs-query.dto'; +import { + ScrapeRunListResponseDto, + ScrapeRunResponseDto, + ScraperStatusResponseDto, +} from './dtos/responses/scrape-run-response.dto'; + +const DEFAULT_PAGE_SIZE = 20; + +@ApiTags('admin') +@ApiSecurity('X-ADMIN-API-KEY') +@Controller('admin') +@UseGuards(AdminApiKeyGuard) +export class ScrapeRunsController { + constructor(private readonly scrapeRunsService: ScrapeRunsService) {} + + @Get('scrapers') + @ApiOkResponse({ type: NativeResponseDto }) + async getScraperStatuses(): Promise> { + const statuses = await this.scrapeRunsService.getScraperStatuses(); + const data = statuses.map(status => ({ + type: status.type, + latestRun: status.latestRun && ScrapeRunResponseDto.from(status.latestRun), + lastSucceededRun: + status.lastSucceededRun && ScrapeRunResponseDto.from(status.lastSucceededRun), + })); + return new NativeResponseDto(data); + } + + @Get('scrape-runs') + @ApiOkResponse({ type: NativeResponseDto }) + async getRuns( + @Query() query: ListScrapeRunsQueryDto, + ): Promise> { + const result = await this.scrapeRunsService.getRuns({ + type: query.type, + status: query.status, + cursor: query.cursor, + limit: query.limit ?? DEFAULT_PAGE_SIZE, + }); + return new NativeResponseDto({ + items: result.runs.map(ScrapeRunResponseDto.from), + nextCursor: result.nextCursor, + }); + } + + @Get('scrape-runs/:id') + @ApiOkResponse({ type: NativeResponseDto }) + async getRun( + @Param('id', ParseIntPipe) id: number, + ): Promise> { + const run = await this.scrapeRunsService.getRun(id); + return new NativeResponseDto(ScrapeRunResponseDto.from(run)); + } + + @Post('scrape-runs') + @HttpCode(HttpStatus.ACCEPTED) + @ApiAcceptedResponse({ + type: NativeResponseDto, + description: '수집 요청이 대기열에 등록됨. 배치가 폴링해 실행한다', + }) + async requestRun( + @Body() body: CreateScrapeRunRequestDto, + ): Promise> { + const run = await this.scrapeRunsService.requestRun(body.type); + return new NativeResponseDto(ScrapeRunResponseDto.from(run), 'Accepted', HttpStatus.ACCEPTED); + } +} diff --git a/services/api/app/src/api/admin/scrape-runs/scrape-runs.module.ts b/services/api/app/src/api/admin/scrape-runs/scrape-runs.module.ts new file mode 100644 index 0000000..5a65dae --- /dev/null +++ b/services/api/app/src/api/admin/scrape-runs/scrape-runs.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { AdminApiKeyGuard } from 'src/api/admin/common/guards/admin-api-key.guard'; +import { ScrapeRunsService } from 'src/api/admin/scrape-runs/application/scrape-runs.service'; +import { ScrapeRun } from 'src/api/admin/scrape-runs/domain/entities/scrape-run.entity'; +import { ScrapeRunRepository } from 'src/api/admin/scrape-runs/infrastructure/scrape-run.repository'; +import { ScrapeRunsController } from 'src/api/admin/scrape-runs/presentation/scrape-runs.controller'; + +@Module({ + imports: [TypeOrmModule.forFeature([ScrapeRun])], + controllers: [ScrapeRunsController], + providers: [ScrapeRunsService, ScrapeRunRepository, AdminApiKeyGuard], +}) +export class ScrapeRunsModule {} diff --git a/services/api/app/src/app.module.ts b/services/api/app/src/app.module.ts index 68db0f5..7fbaa8e 100644 --- a/services/api/app/src/app.module.ts +++ b/services/api/app/src/app.module.ts @@ -14,6 +14,7 @@ import { NoticesModule } from './api/public/notices/notices.module'; import { SchedulesModule } from './api/public/schedules/schedules.module'; import { ShuttlesModule } from './api/public/shuttles/shuttles.module'; import { HealthModule } from './api/internal/health/health.module'; +import { ScrapeRunsModule } from './api/admin/scrape-runs/scrape-runs.module'; @Module({ imports: [ @@ -33,6 +34,7 @@ import { HealthModule } from './api/internal/health/health.module'; SchedulesModule, ShuttlesModule, HealthModule, + ScrapeRunsModule, ], providers: [ { diff --git a/services/api/app/src/main.ts b/services/api/app/src/main.ts index dd581f6..deff9fc 100644 --- a/services/api/app/src/main.ts +++ b/services/api/app/src/main.ts @@ -46,6 +46,15 @@ async function bootstrap() { }, 'X-USER-ID', ) + .addApiKey( + { + type: 'apiKey', + name: 'X-ADMIN-API-KEY', + in: 'header', + description: 'admin API 키 (ADMIN_API_KEY 환경변수)', + }, + 'X-ADMIN-API-KEY', + ) .addSecurityRequirements('X-USER-ID') .build(); diff --git a/services/api/batch/src/batch/batch.module.ts b/services/api/batch/src/batch/batch.module.ts index 6d026e8..9c44fa8 100644 --- a/services/api/batch/src/batch/batch.module.ts +++ b/services/api/batch/src/batch/batch.module.ts @@ -14,16 +14,19 @@ import { ShuttleClient } from './jobs/shuttle/shuttle.client'; import { ShuttleJob } from './jobs/shuttle/shuttle.job'; import { ShuttleParser } from './jobs/shuttle/shuttle.parser'; import { ShuttleRepository } from './jobs/shuttle/shuttle.repository'; +import { ScrapeRun } from './scrape-run/domain/scrape-run.entity'; +import { ScrapeRunRepository } from './scrape-run/scrape-run.repository'; @Module({ imports: [ ConfigModule.forRoot({ isGlobal: true }), ScheduleModule.forRoot(), DatabaseModule, - TypeOrmModule.forFeature([ShuttleTimetable]), + TypeOrmModule.forFeature([ShuttleTimetable, ScrapeRun]), ], providers: [ BatchService, + ScrapeRunRepository, FetchHttpClient, ShuttleClient, ShuttleParser, diff --git a/services/api/batch/src/batch/batch.service.spec.ts b/services/api/batch/src/batch/batch.service.spec.ts index f1464be..b9339c8 100644 --- a/services/api/batch/src/batch/batch.service.spec.ts +++ b/services/api/batch/src/batch/batch.service.spec.ts @@ -1,56 +1,201 @@ import { BatchService } from './batch.service'; +import { BatchJob } from './jobs/batch-job.interface'; +import { + ClaimedScrapeRun, + ScrapeRunRepository, +} from './scrape-run/scrape-run.repository'; describe('BatchService', () => { + let nextRunId: number; + let repository: { + start: jest.Mock; + claimPending: jest.Mock; + succeed: jest.Mock; + fail: jest.Mock; + failInterrupted: jest.Mock; + }; + + const createService = (jobs: BatchJob[]) => + new BatchService(jobs, repository as unknown as ScrapeRunRepository); + + const queuePending = (...runs: ClaimedScrapeRun[]) => { + runs.forEach((run) => repository.claimPending.mockResolvedValueOnce(run)); + }; + beforeEach(() => { + nextRunId = 1; + repository = { + start: jest.fn(() => Promise.resolve(nextRunId++)), + claimPending: jest.fn().mockResolvedValue(null), + succeed: jest.fn().mockResolvedValue(undefined), + fail: jest.fn().mockResolvedValue(undefined), + failInterrupted: jest.fn().mockResolvedValue(0), + }; jest.spyOn(console, 'warn').mockImplementation(() => undefined); + jest.spyOn(console, 'error').mockImplementation(() => undefined); }); afterEach(() => { jest.restoreAllMocks(); }); - it('이전 실행의 잡이 남아 있으면 새 실행을 건너뛴다', async () => { - const pendingJobs: Array<() => void> = []; - const firstJobRun = jest.fn().mockResolvedValue(undefined); - const secondJobRun = jest.fn( - () => new Promise((resolve) => pendingJobs.push(resolve)), - ); - const service = new BatchService([ - { name: 'first', run: firstJobRun }, - { name: 'second', run: secondJobRun }, - ]); - - const firstRun = service.run(); - await new Promise((resolve) => process.nextTick(resolve)); - const secondRun = service.run(); - await new Promise((resolve) => process.nextTick(resolve)); - - expect(firstJobRun).toHaveBeenCalledTimes(1); - - pendingJobs.forEach((resolve) => resolve()); - await Promise.all([firstRun, secondRun]); + describe('run (cron)', () => { + it('이전 실행의 잡이 남아 있으면 새 실행을 건너뛴다', async () => { + const pendingJobs: Array<() => void> = []; + const firstJobRun = jest.fn().mockResolvedValue(undefined); + const secondJobRun = jest.fn( + () => new Promise((resolve) => pendingJobs.push(resolve)), + ); + const service = createService([ + { name: 'first', run: firstJobRun }, + { name: 'second', run: secondJobRun }, + ]); + + const firstRun = service.run(); + await new Promise((resolve) => setImmediate(resolve)); + const secondRun = service.run(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(firstJobRun).toHaveBeenCalledTimes(1); + + pendingJobs.forEach((resolve) => resolve()); + await Promise.all([firstRun, secondRun]); + }); + + it('실행이 끝나면 다음 실행을 진행한다', async () => { + const jobRun = jest.fn().mockResolvedValue(undefined); + const service = createService([{ name: 'job', run: jobRun }]); + + await service.run(); + await service.run(); + + expect(jobRun).toHaveBeenCalledTimes(2); + }); + + it('잡마다 cron run을 기록하고 성공 처리한다', async () => { + const service = createService([ + { name: 'shuttle', run: jest.fn().mockResolvedValue(undefined) }, + ]); + + await service.run(); + + expect(repository.start).toHaveBeenCalledWith('shuttle', 'cron'); + expect(repository.succeed).toHaveBeenCalledWith(1); + expect(repository.fail).not.toHaveBeenCalled(); + }); + + it('잡이 실패하면 run을 실패 처리하고 다음 잡을 실행한다', async () => { + const nextJobRun = jest.fn().mockResolvedValue(undefined); + const service = createService([ + { + name: 'failing', + run: jest.fn().mockRejectedValue(new Error('fail')), + }, + { name: 'next', run: nextJobRun }, + ]); + + await service.run(); + + expect(repository.fail).toHaveBeenCalledWith(1, 'fail'); + expect(repository.succeed).toHaveBeenCalledWith(2); + expect(nextJobRun).toHaveBeenCalledTimes(1); + }); + + it('같은 타입의 run이 이미 대기/실행 중이면 해당 잡을 건너뛴다', async () => { + repository.start.mockResolvedValueOnce(null); + const skippedJobRun = jest.fn().mockResolvedValue(undefined); + const nextJobRun = jest.fn().mockResolvedValue(undefined); + const service = createService([ + { name: 'skipped', run: skippedJobRun }, + { name: 'next', run: nextJobRun }, + ]); + + await service.run(); + + expect(skippedJobRun).not.toHaveBeenCalled(); + expect(nextJobRun).toHaveBeenCalledTimes(1); + }); }); - it('실행이 끝나면 다음 실행을 진행한다', async () => { - const jobRun = jest.fn().mockResolvedValue(undefined); - const service = new BatchService([{ name: 'job', run: jobRun }]); + describe('runPending (수동 실행)', () => { + it('대기 중인 run을 모두 가져와 해당 잡을 실행한다', async () => { + const shuttleRun = jest.fn().mockResolvedValue(undefined); + const noticeRun = jest.fn().mockResolvedValue(undefined); + const service = createService([ + { name: 'shuttle', run: shuttleRun }, + { name: 'notice', run: noticeRun }, + ]); + queuePending({ id: 10, type: 'shuttle' }, { id: 11, type: 'notice' }); + + await service.runPending(); + + expect(shuttleRun).toHaveBeenCalledTimes(1); + expect(noticeRun).toHaveBeenCalledTimes(1); + expect(repository.succeed).toHaveBeenCalledWith(10); + expect(repository.succeed).toHaveBeenCalledWith(11); + }); + + it('잡이 실패해도 run을 실패 처리하고 다음 대기 run을 실행한다', async () => { + const nextRun = jest.fn().mockResolvedValue(undefined); + const service = createService([ + { + name: 'shuttle', + run: jest.fn().mockRejectedValue(new Error('boom')), + }, + { name: 'notice', run: nextRun }, + ]); + queuePending({ id: 10, type: 'shuttle' }, { id: 11, type: 'notice' }); + + await service.runPending(); - await service.run(); - await service.run(); + expect(repository.fail).toHaveBeenCalledWith(10, 'boom'); + expect(nextRun).toHaveBeenCalledTimes(1); + expect(repository.succeed).toHaveBeenCalledWith(11); + }); - expect(jobRun).toHaveBeenCalledTimes(2); + it('등록되지 않은 타입의 run은 실패 처리한다', async () => { + const service = createService([]); + queuePending({ id: 10, type: 'unknown' }); + + await service.runPending(); + + expect(repository.fail).toHaveBeenCalledWith( + 10, + '알 수 없는 잡 타입: unknown', + ); + }); + + it('이전 폴링이 진행 중이면 새 폴링을 건너뛴다', async () => { + let release: () => void = () => undefined; + const jobRun = jest.fn( + () => new Promise((resolve) => (release = resolve)), + ); + const service = createService([{ name: 'shuttle', run: jobRun }]); + queuePending({ id: 10, type: 'shuttle' }); + + const firstPoll = service.runPending(); + await new Promise((resolve) => setImmediate(resolve)); + await service.runPending(); + + expect(repository.claimPending).toHaveBeenCalledTimes(1); + + release(); + await firstPoll; + }); }); - it('잡이 실패해도 다음 잡을 실행한다', async () => { - jest.spyOn(console, 'error').mockImplementation(() => undefined); - const nextJobRun = jest.fn().mockResolvedValue(undefined); - const service = new BatchService([ - { name: 'failing', run: jest.fn().mockRejectedValue(new Error('fail')) }, - { name: 'next', run: nextJobRun }, - ]); + describe('onApplicationBootstrap', () => { + it('중단된 run을 실패 처리한 뒤 cron 실행을 시작한다', async () => { + const jobRun = jest.fn().mockResolvedValue(undefined); + const service = createService([{ name: 'shuttle', run: jobRun }]); - await service.run(); + await service.onApplicationBootstrap(); - expect(nextJobRun).toHaveBeenCalledTimes(1); + expect(repository.failInterrupted).toHaveBeenCalledTimes(1); + expect( + repository.failInterrupted.mock.invocationCallOrder[0], + ).toBeLessThan(repository.start.mock.invocationCallOrder[0]); + expect(jobRun).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/services/api/batch/src/batch/batch.service.ts b/services/api/batch/src/batch/batch.service.ts index 67aead8..67022a1 100644 --- a/services/api/batch/src/batch/batch.service.ts +++ b/services/api/batch/src/batch/batch.service.ts @@ -1,16 +1,24 @@ import { Inject, Injectable, OnApplicationBootstrap } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; +import { Cron, Interval } from '@nestjs/schedule'; import { BATCH_JOBS, BatchJob } from './jobs/batch-job.interface'; +import { ScrapeRunRepository } from './scrape-run/scrape-run.repository'; + +const MANUAL_RUN_POLL_INTERVAL_MS = 10_000; @Injectable() export class BatchService implements OnApplicationBootstrap { private readonly name = 'BatchService'; + private readonly jobsByName: Map; private isRunning = false; + private isPolling = false; constructor( @Inject(BATCH_JOBS) private readonly jobs: BatchJob[], - ) {} + private readonly scrapeRunRepository: ScrapeRunRepository, + ) { + this.jobsByName = new Map(jobs.map((job) => [job.name, job])); + } @Cron('0 0 * * * *') // 매 정각마다 실행 async run(): Promise { @@ -26,7 +34,14 @@ export class BatchService implements OnApplicationBootstrap { try { for (const job of this.jobs) { try { - await job.run(); + const runId = await this.scrapeRunRepository.start(job.name, 'cron'); + if (runId === null) { + console.warn( + `[${this.name}] 이미 대기/실행 중인 run이 있어 건너뜁니다: ${job.name}`, + ); + continue; + } + await this.execute(runId, job); } catch (error) { // TODO: 에러 로깅 및 알림 시스템 연동 console.error( @@ -40,7 +55,62 @@ export class BatchService implements OnApplicationBootstrap { } } + /** admin API로 요청된 수동 실행(pending run)을 가져와 실행한다. */ + @Interval(MANUAL_RUN_POLL_INTERVAL_MS) + async runPending(): Promise { + if (this.isPolling) return; + + this.isPolling = true; + + try { + let claimed = await this.scrapeRunRepository.claimPending(); + while (claimed) { + const job = this.jobsByName.get(claimed.type); + if (job) { + try { + await this.execute(claimed.id, job); + } catch (error) { + console.error( + `[${this.name}] 수동 실행 잡 에러 발생: ${job.name}`, + error, + ); + } + } else { + await this.scrapeRunRepository.fail( + claimed.id, + `알 수 없는 잡 타입: ${claimed.type}`, + ); + } + claimed = await this.scrapeRunRepository.claimPending(); + } + } catch (error) { + console.error(`[${this.name}] 수동 실행 처리 중 에러 발생`, error); + } finally { + this.isPolling = false; + } + } + async onApplicationBootstrap(): Promise { + const interrupted = await this.scrapeRunRepository.failInterrupted(); + if (interrupted > 0) { + console.warn( + `[${this.name}] 중단된 run ${interrupted}건을 실패 처리했습니다.`, + ); + } await this.run(); } + + private async execute(runId: number, job: BatchJob): Promise { + try { + await job.run(); + } catch (error) { + await this.scrapeRunRepository.fail(runId, toErrorMessage(error)); + throw error; + } + await this.scrapeRunRepository.succeed(runId); + } +} + +function toErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); } diff --git a/services/api/batch/src/batch/scrape-run/domain/scrape-run.entity.ts b/services/api/batch/src/batch/scrape-run/domain/scrape-run.entity.ts new file mode 100644 index 0000000..fe3b70a --- /dev/null +++ b/services/api/batch/src/batch/scrape-run/domain/scrape-run.entity.ts @@ -0,0 +1,31 @@ +import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; + +export type ScrapeRunTrigger = 'cron' | 'manual'; +export type ScrapeRunStatus = 'pending' | 'running' | 'succeeded' | 'failed'; + +@Entity('scrape_run') +export class ScrapeRun { + @PrimaryGeneratedColumn() + id: number; + + @Column({ length: 30 }) + type: string; + + @Column({ length: 10 }) + trigger: ScrapeRunTrigger; + + @Column({ length: 10 }) + status: ScrapeRunStatus; + + @Column({ name: 'error_message', type: 'text', nullable: true }) + errorMessage: string | null; + + @Column({ name: 'created_at', type: 'timestamptz' }) + createdAt: Date; + + @Column({ name: 'started_at', type: 'timestamptz', nullable: true }) + startedAt: Date | null; + + @Column({ name: 'finished_at', type: 'timestamptz', nullable: true }) + finishedAt: Date | null; +} diff --git a/services/api/batch/src/batch/scrape-run/scrape-run.repository.ts b/services/api/batch/src/batch/scrape-run/scrape-run.repository.ts new file mode 100644 index 0000000..a993be1 --- /dev/null +++ b/services/api/batch/src/batch/scrape-run/scrape-run.repository.ts @@ -0,0 +1,91 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { QueryFailedError, Repository } from 'typeorm'; +import { ScrapeRun, ScrapeRunTrigger } from './domain/scrape-run.entity'; + +const UNIQUE_VIOLATION = '23505'; + +export interface ClaimedScrapeRun { + id: number; + type: string; +} + +@Injectable() +export class ScrapeRunRepository { + constructor( + @InjectRepository(ScrapeRun) + private readonly repository: Repository, + ) {} + + /** + * 바로 실행 상태의 run을 만든다. + * 같은 타입의 run이 이미 대기/실행 중이면 null을 반환한다. + */ + async start(type: string, trigger: ScrapeRunTrigger): Promise { + try { + const rows: { id: number }[] = await this.repository.query( + `INSERT INTO scrape_run (type, trigger, status, started_at) + VALUES ($1, $2, 'running', now()) + RETURNING id`, + [type, trigger], + ); + return Number(rows[0].id); + } catch (error) { + if (isUniqueViolation(error)) return null; + throw error; + } + } + + /** 가장 오래된 대기 중 run 하나를 실행 상태로 바꾸고 반환한다. */ + async claimPending(): Promise { + const [rows]: [{ id: number; type: string }[], number] = + await this.repository.query( + `UPDATE scrape_run + SET status = 'running', started_at = now() + WHERE id = ( + SELECT id FROM scrape_run + WHERE status = 'pending' + ORDER BY id + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + RETURNING id, type`, + ); + if (!rows || rows.length === 0) return null; + return { id: Number(rows[0].id), type: rows[0].type }; + } + + async succeed(id: number): Promise { + await this.repository.query( + `UPDATE scrape_run SET status = 'succeeded', finished_at = now() WHERE id = $1`, + [id], + ); + } + + async fail(id: number, errorMessage: string): Promise { + await this.repository.query( + `UPDATE scrape_run + SET status = 'failed', finished_at = now(), error_message = $2 + WHERE id = $1`, + [id, errorMessage], + ); + } + + /** 프로세스 종료로 실행 중 상태에 남은 run을 실패 처리한다. */ + async failInterrupted(): Promise { + const [, affected]: [unknown, number] = await this.repository.query( + `UPDATE scrape_run + SET status = 'failed', finished_at = now(), + error_message = '배치 프로세스 재시작으로 중단됨' + WHERE status = 'running'`, + ); + return affected; + } +} + +function isUniqueViolation(error: unknown): boolean { + return ( + error instanceof QueryFailedError && + (error.driverError as { code?: string }).code === UNIQUE_VIOLATION + ); +} diff --git a/services/api/nginx/prod.conf b/services/api/nginx/prod.conf index c212f05..17c01c3 100644 --- a/services/api/nginx/prod.conf +++ b/services/api/nginx/prod.conf @@ -30,23 +30,7 @@ server { ssl_certificate /etc/letsencrypt/live/api.connectgnu.kro.kr-0001/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/api.connectgnu.kro.kr-0001/privkey.pem; - # Swagger 경로 - location /api/api-docs { - proxy_pass http://app; - proxy_http_version 1.1; - proxy_pass_request_headers on; - proxy_set_header Connection ""; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $http_host; - proxy_set_header X-NginX-Proxy true; - - allow 210.121.177.238; - deny all; - } - - # API 경로 + # API 경로 (Swagger /api/api-docs 포함) location /api { proxy_pass http://app; proxy_http_version 1.1; diff --git a/supabase/migrations/20260925000000_add_scrape_run.sql b/supabase/migrations/20260925000000_add_scrape_run.sql new file mode 100644 index 0000000..8baa2b3 --- /dev/null +++ b/supabase/migrations/20260925000000_add_scrape_run.sql @@ -0,0 +1,20 @@ +create table scrape_run ( + id bigint generated always as identity primary key, + type varchar(30) not null + check (type in ('shuttle', 'notice', 'cafeteria', 'academic-calendar')), + trigger varchar(10) not null + check (trigger in ('cron', 'manual')), + status varchar(10) not null default 'pending' + check (status in ('pending', 'running', 'succeeded', 'failed')), + error_message text, + created_at timestamptz not null default now(), + started_at timestamptz, + finished_at timestamptz +); + +-- 타입별로 대기/실행 중인 run은 하나만 존재할 수 있다 (cron·수동 실행 공통 락) +create unique index scrape_run_active_type_uq + on scrape_run (type) + where status in ('pending', 'running'); + +create index scrape_run_type_id_idx on scrape_run (type, id desc);