CCM-13304: Use named query

Author: simonlabarere

infrastructure/terraform/components/dl/athena_named_query_daily_report.tf

@@ -0,0 +1,6 @@
+resource "aws_athena_named_query" "daily_report" {
+  name      = "daily_report"
+  workgroup = aws_athena_workgroup.reporting.id
+  database  = aws_glue_catalog_database.reporting.name
+  query     = file("${path.module}/scripts/sql/reports/daily_report.sql")
+}

infrastructure/terraform/components/dl/module_lambda_report_generator.tf

@@ -35,8 +35,7 @@ module "report_generator" {
   log_subscription_role_arn = local.acct.log_subscription_role_arn
 
   lambda_env_vars = {
-    "ATHENA_WORKGROUP"              = aws_athena_workgroup.reporting.name
-    "ATHENA_DATABASE"               = aws_glue_catalog_database.reporting.name
+    "ATHENA_NAMED_QUERY_ID"         = aws_athena_named_query.daily_report.name
     "EVENT_PUBLISHER_EVENT_BUS_ARN" = aws_cloudwatch_event_bus.main.arn
     "EVENT_PUBLISHER_DLQ_URL"       = module.sqs_event_publisher_errors.sqs_queue_url
     "MAX_POLL_LIMIT"                = var.athena_query_max_polling_attempts
@@ -153,17 +152,4 @@ data "aws_iam_policy_document" "report_generator_lambda" {
       module.sqs_event_publisher_errors.sqs_queue_arn,
     ]
   }
-
-  statement {
-    sid    = "SQSDLQPermissions"
-    effect = "Allow"
-
-    actions = [
-      "sqs:SendMessage",
-    ]
-
-    resources = [
-      module.sqs_report_generator.sqs_dlq_arn,
-    ]
-  }
 }

…/report-generator/src/queries/report.sql …s/dl/script/sql/reports/daily_report.sqllambdas/report-generator/src/queries/report.sql renamed to infrastructure/terraform/components/dl/script/sql/reports/daily_report.sql lambdas/report-generator/src/queries/report.sql renamed to infrastructure/terraform/components/dl/script/sql/reports/daily_report.sql

@@ -1,7 +1,6 @@
 {
   "dependencies": {
     "@aws-sdk/client-athena": "^3.984.0",
-    "@aws-sdk/client-sqs": "^3.984.0",
     "digital-letters-events": "^0.0.1",
     "utils": "^0.0.1"
   },
@@ -15,7 +14,7 @@
   "name": "nhs-notify-digital-letters-report-generator",
   "private": true,
   "scripts": {
-    "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts && cp -r src/queries dist/",
+    "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "test:unit": "jest",

lambdas/report-generator/package.json

@@ -10,6 +10,7 @@ describe('ReportGenerator', () => {
   let mockReportService: jest.Mocked<IReportService>;
   let reportGenerator: ReportGenerator;
   const reportName = 'completed_communications';
+  const athenaNamedQueryId = 'test-athena-named-query-id';
 
   beforeEach(() => {
     mockLogger = {
@@ -27,6 +28,7 @@ describe('ReportGenerator', () => {
       mockLogger,
       mockReportService,
       reportName,
+      athenaNamedQueryId,
     );
 
     jest.clearAllMocks();
@@ -64,15 +66,16 @@ describe('ReportGenerator', () => {
 
       const result = await reportGenerator.generate(mockEvent);
 
-      expect(fs.readFileSync).toHaveBeenCalledWith(
-        '/var/task/queries/report.sql',
-        'utf8',
-      );
-      expect(mockLogger.info).toHaveBeenCalledWith(
-        'Generating report for sender sender-123 and date 2025-01-15',
-      );
+      expect(mockLogger.info).toHaveBeenCalledWith({
+        description: 'Generating report',
+        senderId: 'sender-123',
+        reportDate: '2025-01-15',
+        athenaNamedQueryId,
+        reportFilePath:
+          'event-reports/sender-123/completed_communications/completed_communications_2025-01-15.csv',
+      });
       expect(mockReportService.generateReport).toHaveBeenCalledWith(
-        mockQuery,
+        athenaNamedQueryId,
         ["'2025-01-15'", "'sender-123'"],
         'event-reports/sender-123/completed_communications/completed_communications_2025-01-15.csv',
       );
@@ -98,24 +101,5 @@ describe('ReportGenerator', () => {
         outcome: 'failed',
       });
     });
-
-    it('should return failed outcome when file read throws error', async () => {
-      const error = new Error('File not found');
-      (fs.readFileSync as jest.Mock).mockImplementation(() => {
-        throw error;
-      });
-
-      const result = await reportGenerator.generate(mockEvent);
-
-      expect(mockLogger.error).toHaveBeenCalledWith({
-        err: error,
-        description: 'Error generating report',
-        senderId: 'sender-123',
-        reportDate: '2025-01-15',
-      });
-      expect(result).toEqual({
-        outcome: 'failed',
-      });
-    });
   });
 });

lambdas/report-generator/src/__tests__/app/report-generator.test.ts

@@ -1,6 +1,5 @@
 import { IReportService, Logger } from 'utils';
 import { GenerateReport } from 'digital-letters-events';
-import fs from 'node:fs';
 
 export type ReportGeneratorOutcome = 'generated' | 'failed';
 
@@ -18,21 +17,25 @@ export class ReportGenerator {
     private readonly logger: Logger,
     private readonly reportService: IReportService,
     private readonly reportName: string,
+    private readonly athenaNamedQueryId: string,
   ) {}
 
   async generate(event: GenerateReport): Promise<ReportGeneratorResult> {
     const { reportDate, senderId } = event.data;
 
     try {
-      const query = fs.readFileSync('/var/task/queries/report.sql', 'utf8');
       const reportFilePath = `event-reports/${senderId}/${this.reportName}/${this.reportName}_${reportDate}.csv`;
 
-      this.logger.info(
-        `Generating report for sender ${senderId} and date ${reportDate}`,
-      );
+      this.logger.info({
+        description: 'Generating report',
+        senderId,
+        reportDate,
+        athenaNamedQueryId: this.athenaNamedQueryId,
+        reportFilePath,
+      });
 
       const location = await this.reportService.generateReport(
-        query,
+        this.athenaNamedQueryId,
         [`'${reportDate}'`, `'${senderId}'`],
         reportFilePath,
       );

lambdas/report-generator/src/app/report-generator.ts

@@ -16,8 +16,7 @@ import { AthenaClient } from '@aws-sdk/client-athena';
 
 export const createContainer = () => {
   const {
-    athenaDatabase,
-    athenaWorkgroup,
+    athenaNamedQueryId,
     eventPublisherDlqUrl,
     eventPublisherEventBusArn,
     maxPollLimit,
@@ -32,8 +31,6 @@ export const createContainer = () => {
 
   const dataRepositoryDependencies: AthenaDataRepositoryDependencies = {
     athenaClient,
-    athenaDatabase,
-    athenaWorkgroup,
   };
 
   const dataRepository = new AthenaDataRepository(dataRepositoryDependencies);
@@ -56,6 +53,7 @@ export const createContainer = () => {
     logger,
     reportService,
     reportName,
+    athenaNamedQueryId,
   );
 
   const eventPublisher = new EventPublisher({

lambdas/report-generator/src/container.ts

@@ -1,8 +1,7 @@
 import { defaultConfigReader } from 'utils';
 
 export type ReportGeneratorConfig = {
-  athenaWorkgroup: string;
-  athenaDatabase: string;
+  athenaNamedQueryId: string;
   eventPublisherEventBusArn: string;
   eventPublisherDlqUrl: string;
   maxPollLimit: number;
@@ -13,8 +12,7 @@ export type ReportGeneratorConfig = {
 
 export function loadConfig(): ReportGeneratorConfig {
   return {
-    athenaWorkgroup: defaultConfigReader.getValue('ATHENA_WORKGROUP'),
-    athenaDatabase: defaultConfigReader.getValue('ATHENA_DATABASE'),
+    athenaNamedQueryId: defaultConfigReader.getValue('ATHENA_NAMED_QUERY_ID'),
     eventPublisherEventBusArn: defaultConfigReader.getValue(
       'EVENT_PUBLISHER_EVENT_BUS_ARN',
     ),

lambdas/report-generator/src/infra/config.ts

@@ -1,4 +1,8 @@
-import { AthenaClient } from '@aws-sdk/client-athena';
+import {
+  AthenaClient,
+  GetNamedQueryCommand,
+  StartQueryExecutionCommand,
+} from '@aws-sdk/client-athena';
 import { mockClient } from 'aws-sdk-client-mock';
 import {
   AthenaDataRepository,
@@ -11,8 +15,6 @@ describe('AthenaDataRepository', () => {
   let repository: AthenaDataRepository;
   const config: AthenaDataRepositoryDependencies = {
     athenaClient: new AthenaClient({}),
-    athenaWorkgroup: 'test-workgroup',
-    athenaDatabase: 'test-database',
   };
 
   beforeEach(() => {
@@ -23,11 +25,19 @@ describe('AthenaDataRepository', () => {
   describe('startQuery', () => {
     it('should start query execution and return query execution ID', async () => {
       const mockQueryExecutionId = 'query-123';
-      athenaClientMock.onAnyCommand().resolves({
+      athenaClientMock.on(GetNamedQueryCommand).resolves({
+        NamedQuery: {
+          Name: 'Test Named Query',
+          Database: 'test-database',
+          WorkGroup: 'test-workgroup',
+          QueryString: 'SELECT * FROM table',
+        },
+      });
+      athenaClientMock.on(StartQueryExecutionCommand).resolves({
         QueryExecutionId: mockQueryExecutionId,
       });
 
-      const result = await repository.startQuery('SELECT * FROM table', [
+      const result = await repository.startQuery('testNamedQueryId', [
         'param1',
         'param2',
       ]);
@@ -38,32 +48,74 @@ describe('AthenaDataRepository', () => {
     it('should send correct parameters to Athena client', async () => {
       const query = 'SELECT * FROM table WHERE id = ?';
       const executionParameters = ['123'];
+      const namedQueryId = 'testNamedQueryId';
+      const testDatabase = 'test-database';
+      const testWorkGroup = 'test-workgroup';
 
-      athenaClientMock.onAnyCommand().resolves({
-        QueryExecutionId: 'query-456',
+      const mockQueryExecutionId = 'query-123';
+      athenaClientMock.on(GetNamedQueryCommand).resolves({
+        NamedQuery: {
+          Name: 'Test Named Query',
+          Database: testDatabase,
+          WorkGroup: testWorkGroup,
+          QueryString: query,
+        },
+      });
+      athenaClientMock.on(StartQueryExecutionCommand).resolves({
+        QueryExecutionId: mockQueryExecutionId,
       });
 
-      await repository.startQuery(query, executionParameters);
+      await repository.startQuery(namedQueryId, executionParameters);
 
       const calls = athenaClientMock.commandCalls(
-        Object.getPrototypeOf(athenaClientMock.calls()[0].args[0]).constructor,
+        Object.getPrototypeOf(athenaClientMock.calls()[1].args[0]).constructor,
       );
       expect(calls[0].args[0].input).toEqual({
         QueryString: query,
-        WorkGroup: 'test-workgroup',
-        QueryExecutionContext: { Database: 'test-database' },
+        WorkGroup: testWorkGroup,
+        QueryExecutionContext: { Database: testDatabase },
         ExecutionParameters: executionParameters,
       });
     });
 
-    it('should handle empty execution parameters', async () => {
-      athenaClientMock.onAnyCommand().resolves({
-        QueryExecutionId: 'query-789',
+    it('should throw an error if named query is not found', async () => {
+      athenaClientMock.on(GetNamedQueryCommand).resolves({});
+
+      await expect(
+        repository.startQuery('nonExistentNamedQuery', ['param']),
+      ).rejects.toThrow(
+        'Named query nonExistentNamedQuery not found or has no SQL.',
+      );
+    });
+
+    it('should throw an error if named query does not specify a database', async () => {
+      athenaClientMock.on(GetNamedQueryCommand).resolves({
+        NamedQuery: {
+          WorkGroup: 'test-workgroup',
+          QueryString: 'SELECT 1',
+        } as any,
       });
 
-      const result = await repository.startQuery('SELECT 1', []);
+      await expect(
+        repository.startQuery('namedQueryWithoutDatabase', ['param']),
+      ).rejects.toThrow(
+        'Named query namedQueryWithoutDatabase does not specify a database.',
+      );
+    });
+
+    it('should throw an error if named query does not specify a workgroup', async () => {
+      athenaClientMock.on(GetNamedQueryCommand).resolves({
+        NamedQuery: {
+          Database: 'test-database',
+          QueryString: 'SELECT 1',
+        } as any,
+      });
 
-      expect(result).toBe('query-789');
+      await expect(
+        repository.startQuery('namedQueryWithoutWorkgroup', ['param']),
+      ).rejects.toThrow(
+        'Named query namedQueryWithoutWorkgroup does not specify a workgroup.',
+      );
     });
 
     it('should propagate Athena client errors', async () => {