VED-1260 Mock PDS lambda rate limiting in ref env

.github/workflows/deploy-backend.yml

@@ -37,6 +37,11 @@ on:
         required: false
         type: boolean
         default: true
+      deploy_mock_pds:
+        description: Deploy the mock PDS Lambda for perf testing. Ref only.
+        required: false
+        type: boolean
+        default: false
       environment:
         required: true
         type: string
@@ -99,6 +104,11 @@ on:
         required: true
         type: boolean
         default: false
+      deploy_mock_pds:
+        description: Deploy the mock PDS Lambda for perf testing. Ref only.
+        required: false
+        type: boolean
+        default: false
       sub_environment:
         type: string
         description: Set the sub environment name e.g. pr-xxx, or green/blue in higher environments
@@ -170,19 +180,44 @@ jobs:
       image_tag_prefix: ${{ inputs.sub_environment }}-
       allow_implicit_tag_prefix_reuse: ${{ inputs.sub_environment == 'internal-dev' || startsWith(inputs.sub_environment, 'pr-') }}
 
+  deploy-mock-pds-image:
+    name: Deploy mock_pds image
+    if: ${{ inputs.deploy_mock_pds && inputs.sub_environment == 'ref' }}
+    uses: ./.github/workflows/deploy-lambda-artifact.yml
+    with:
+      lambda_name: mock_pds
+      tf_var_suffix: mock_pds
+      environment: ${{ inputs.environment }}
+      sub_environment: ${{ inputs.sub_environment }}
+      build_image: ${{ fromJson(inputs.lambda_build_flags).mock_pds || false }}
+      image_version: ${{ fromJson(inputs.lambda_image_overrides).mock_pds || '' }}
+      run_diff_check: ${{ inputs.run_diff_check }}
+      diff_base_sha: ${{ inputs.diff_base_sha }}
+      diff_head_sha: ${{ inputs.diff_head_sha }}
+      lambda_paths: tests/perf_tests/mock_pds/
+      shared_paths: lambdas/shared/src/common/
+      docker_context_path: tests/perf_tests
+      dockerfile_path: tests/perf_tests/mock_pds/Dockerfile
+      ecr_repository: imms-mock-pds-repo
+      image_tag_prefix: ${{ inputs.sub_environment }}-
+      allow_implicit_tag_prefix_reuse: false
+
   terraform-plan:
     permissions:
       actions: read
       id-token: write
       contents: read
     needs:
       - deploy-lambda-images
-    if: ${{ !cancelled() && needs.deploy-lambda-images.result == 'success' }}
+      - deploy-mock-pds-image
+    if: ${{ !cancelled() && needs.deploy-lambda-images.result == 'success' && (needs.deploy-mock-pds-image.result == 'success' || needs.deploy-mock-pds-image.result == 'skipped') }}
     outputs:
       image_uris_json: ${{ steps.lambda-images.outputs.image_uris_json }}
     runs-on: ubuntu-latest
     environment:
       name: ${{ inputs.environment }}
+    env:
+      TF_VAR_mock_pds_enabled: ${{ inputs.deploy_mock_pds && inputs.sub_environment == 'ref' }}
     steps:
       - name: Checkout
         uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
@@ -271,6 +306,8 @@ jobs:
     runs-on: ubuntu-latest
     environment:
       name: ${{ inputs.environment }}
+    env:
+      TF_VAR_mock_pds_enabled: ${{ inputs.deploy_mock_pds && inputs.sub_environment == 'ref' }}
     steps:
       - name: Checkout
         uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98

.github/workflows/quality-checks.yml

@@ -184,6 +184,18 @@ jobs:
           poetry run coverage run --source=src -m unittest discover || echo "mns_publisher tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../mns_publisher-coverage.xml
 
+      - name: Run unittest with coverage-mock-pds
+        working-directory: tests/perf_tests/mock_pds
+        id: mock_pds
+        env:
+          POETRY_INSTALLER_ONLY_BINARY: ":all:"
+          PYTHONPATH: ${{ github.workspace }}/tests/perf_tests/mock_pds/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run --source=src -m unittest discover -s tests -v || echo "mock_pds tests failed" >> ../../../failed_tests.txt
+          poetry run coverage xml -o ../../../mock_pds-coverage.xml
+
       - name: Run unittest with coverage-mns-subscription
         working-directory: lambdas/mns_subscription
         id: mns_subscription

README.md

@@ -62,9 +62,10 @@ GitHub Actions for our entire pipeline.
 
 ### Tests
 
-| Folder           | Description                                                                   |
-| ---------------- | ----------------------------------------------------------------------------- |
-| `e2e_automation` | End-to-end tests executed during PR pipelines using the pytest-bdd framework. |
+| Folder           | Description                                                                                               |
+| ---------------- | --------------------------------------------------------------------------------------------------------- |
+| `e2e_automation` | End-to-end tests executed during PR pipelines using the pytest-bdd framework.                             |
+| `perf_tests`     | Locust performance tests and the test-only mock PDS Lambda package used for rate-limit validation in ref. |
 
 ---
 

infrastructure/account/lambda_ecr_repos.tf

@@ -38,6 +38,10 @@ locals {
       name                = "imms-mns-publisher-repo"
       lambda_source_names = ["*-mns-publisher-lambda"]
     }
+    mock_pds = {
+      name                = "imms-mock-pds-repo"
+      lambda_source_names = ["*-mock-pds-lambda"]
+    }
     ack_backend = {
       name                = "imms-ackbackend-repo"
       lambda_source_names = ["*-ack-lambda"]

infrastructure/instance/environments/dev/pr/variables.tfvars

@@ -6,3 +6,4 @@ mns_environment                   = "dev"
 error_alarm_notifications_enabled = false
 create_mesh_processor             = false
 has_sub_environment_scope         = true
+mock_pds_enabled                  = false

infrastructure/instance/environments/dev/ref/variables.tfvars

@@ -3,6 +3,9 @@ immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "ref"
 mns_environment                   = "dev"
+mock_pds_enabled                  = false
+mock_pds_average_rate_limit       = 125
+mock_pds_spike_rate_limit         = 450
 error_alarm_notifications_enabled = true
 create_mesh_processor             = false
 has_sub_environment_scope         = true

infrastructure/instance/mock_pds.tf

@@ -0,0 +1,161 @@
+locals {
+  mock_pds_lambda_name = "${local.short_prefix}-mock-pds-lambda"
+}
+
+resource "aws_iam_role" "mock_pds_lambda_exec_role" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  name = "${local.mock_pds_lambda_name}-exec-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Effect = "Allow",
+      Sid    = "",
+      Principal = {
+        Service = "lambda.amazonaws.com"
+      },
+      Action = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_policy" "mock_pds_lambda_exec_policy" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  name = "${local.mock_pds_lambda_name}-exec-policy"
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "logs:CreateLogGroup",
+          "logs:CreateLogStream",
+          "logs:PutLogEvents"
+        ]
+        Resource = "arn:aws:logs:${var.aws_region}:${var.immunisation_account_id}:log-group:/aws/lambda/${local.mock_pds_lambda_name}:*"
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "ec2:CreateNetworkInterface",
+          "ec2:DescribeNetworkInterfaces",
+          "ec2:DeleteNetworkInterface"
+        ],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_policy" "mock_pds_lambda_kms_access_policy" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  name        = "${local.mock_pds_lambda_name}-kms-policy"
+  description = "Allow mock PDS Lambda to decrypt environment variables"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "kms:Decrypt"
+        ]
+        Resource = data.aws_kms_key.existing_lambda_encryption_key.arn
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "mock_pds_lambda_exec_policy_attachment" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  role       = aws_iam_role.mock_pds_lambda_exec_role[0].name
+  policy_arn = aws_iam_policy.mock_pds_lambda_exec_policy[0].arn
+}
+
+resource "aws_iam_role_policy_attachment" "mock_pds_lambda_kms_policy_attachment" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  role       = aws_iam_role.mock_pds_lambda_exec_role[0].name
+  policy_arn = aws_iam_policy.mock_pds_lambda_kms_access_policy[0].arn
+}
+
+resource "aws_cloudwatch_log_group" "mock_pds_lambda_log_group" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  name              = "/aws/lambda/${local.mock_pds_lambda_name}"
+  retention_in_days = 30
+}
+
+resource "aws_lambda_function" "mock_pds_lambda" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  function_name = local.mock_pds_lambda_name
+  role          = aws_iam_role.mock_pds_lambda_exec_role[0].arn
+  package_type  = "Image"
+  image_uri     = var.mock_pds_image_uri
+  architectures = ["x86_64"]
+  timeout       = 30
+
+  vpc_config {
+    subnet_ids         = local.private_subnet_ids
+    security_group_ids = [data.aws_security_group.existing_securitygroup.id]
+  }
+
+  environment {
+    variables = {
+      REDIS_HOST                      = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address
+      REDIS_PORT                      = tostring(data.aws_elasticache_cluster.existing_redis.port)
+      MOCK_PDS_AVERAGE_LIMIT          = tostring(var.mock_pds_average_rate_limit)
+      MOCK_PDS_AVERAGE_WINDOW_SECONDS = tostring(var.mock_pds_average_window_seconds)
+      MOCK_PDS_SPIKE_LIMIT            = tostring(var.mock_pds_spike_rate_limit)
+      MOCK_PDS_SPIKE_WINDOW_SECONDS   = tostring(var.mock_pds_spike_window_seconds)
+      MOCK_PDS_GP_ODS_CODE            = var.mock_pds_gp_ods_code
+    }
+  }
+
+  kms_key_arn = data.aws_kms_key.existing_lambda_encryption_key.arn
+
+  depends_on = [
+    aws_cloudwatch_log_group.mock_pds_lambda_log_group,
+    aws_iam_policy.mock_pds_lambda_exec_policy
+  ]
+}
+
+resource "aws_lambda_function_url" "mock_pds_lambda_url" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  function_name      = aws_lambda_function.mock_pds_lambda[0].function_name
+  authorization_type = "NONE"
+}
+
+resource "aws_lambda_permission" "mock_pds_lambda_url_invoke" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  statement_id           = "AllowPublicInvokeFunctionUrl"
+  action                 = "lambda:InvokeFunctionUrl"
+  function_name          = aws_lambda_function.mock_pds_lambda[0].function_name
+  principal              = "*"
+  function_url_auth_type = "NONE"
+}
+
+resource "aws_cloudwatch_log_metric_filter" "mock_pds_throttle_logs" {
+  count = var.mock_pds_enabled ? 1 : 0
+
+  name           = "${local.short_prefix}-MockPdsThrottleLogs"
+  pattern        = "Mock PDS rate limit exceeded"
+  log_group_name = aws_cloudwatch_log_group.mock_pds_lambda_log_group[0].name
+
+  metric_transformation {
+    name      = "${local.short_prefix}-MockPdsThrottleRequests"
+    namespace = "${local.short_prefix}-MockPds"
+    value     = "1"
+  }
+}
+
+output "mock_pds_function_url" {
+  value       = var.mock_pds_enabled ? aws_lambda_function_url.mock_pds_lambda_url[0].function_url : null
+  description = "Function URL for the mock PDS endpoint."
+}

infrastructure/instance/variables.tf

@@ -79,6 +79,42 @@ variable "mns_environment" {
   default = "int"
 }
 
+variable "mock_pds_enabled" {
+  description = "Enable the mock PDS Lambda in test environments."
+  type        = bool
+  default     = false
+}
+
+variable "mock_pds_average_rate_limit" {
+  description = "Average mock PDS request rate, in requests per second."
+  type        = number
+  default     = 125
+}
+
+variable "mock_pds_average_window_seconds" {
+  description = "Average mock PDS rate limiting window in seconds."
+  type        = number
+  default     = 60
+}
+
+variable "mock_pds_spike_rate_limit" {
+  description = "Spike mock PDS request rate, in requests per second."
+  type        = number
+  default     = 450
+}
+
+variable "mock_pds_spike_window_seconds" {
+  description = "Spike mock PDS rate limiting window in seconds."
+  type        = number
+  default     = 1
+}
+
+variable "mock_pds_gp_ods_code" {
+  description = "Deterministic GP ODS code returned by the mock PDS service."
+  type        = string
+  default     = "Y12345"
+}
+
 variable "mesh_no_invocation_period_seconds" {
   description = "The maximum duration the MESH Processor Lambda can go without being invoked before the no-invocation alarm is triggered."
   type        = number
@@ -136,6 +172,17 @@ variable "ack_backend_image_uri" {
   }
 }
 
+variable "mock_pds_image_uri" {
+  description = "Immutable URI of the mock PDS Lambda container image in ECR. Required when mock_pds_enabled is true; supplied by CI/CD."
+  type        = string
+  default     = ""
+
+  validation {
+    condition     = !var.mock_pds_enabled || trimspace(var.mock_pds_image_uri) != ""
+    error_message = "mock_pds_image_uri must be provided when mock_pds_enabled is true."
+  }
+}
+
 variable "batch_processor_filter_image_uri" {
   description = "Immutable URI of the batch processor filter Lambda container image in ECR. Must be supplied by CI/CD."
   type        = string

tests/perf_tests/Makefile

@@ -1,7 +1,31 @@
+LOCUST_FILE ?= src/locustfile.py
+LOCUST ?= poetry run locust -f $(LOCUST_FILE)
+MOCK_LOCUST_FILE ?= src/locustfile_pds_rate_limit.py
+MOCK_LOCUST ?= poetry run locust -f $(MOCK_LOCUST_FILE)
+
 test:
-	poetry run locust -f src/locustfile.py
+	$(LOCUST)
 
 test-read-only:
-	poetry run locust -f src/locustfile.py SearchUser
+	$(LOCUST) SearchUser
+
+mockpds:
+	$(MOCK_LOCUST) MockPdsUser
+
+mockpdstest-average:
+	PERF_LOAD_PROFILE=average PERF_MOCK_PDS_AVERAGE_RPS=125 $(MOCK_LOCUST) MockPdsUser
+
+mockpdstest-boundary:
+	PERF_LOAD_PROFILE=average PERF_MOCK_PDS_AVERAGE_RPS=130 $(MOCK_LOCUST) MockPdsUser
+
+mockpdstest-spike:
+	PERF_LOAD_PROFILE=spike $(MOCK_LOCUST) MockPdsUser
+
+mockpdstest-ui:
+	@if [ "$(PERF_LOAD_PROFILE)" != "average" ] && [ "$(PERF_LOAD_PROFILE)" != "spike" ]; then \
+		echo "PERF_LOAD_PROFILE must be set to average or spike"; \
+		exit 1; \
+	fi
+	PERF_LOAD_PROFILE=$(PERF_LOAD_PROFILE) $(MOCK_LOCUST) MockPdsUser
 
-.PHONY: test test-read-only
+.PHONY: test test-read-only mockpds mockpdstest-average mockpdstest-boundary mockpdstest-spike mockpdstest-ui