From 99c96126aeddc1cf5cf726aec24f5fe8e72a068c Mon Sep 17 00:00:00 2001 From: amarauzoma Date: Tue, 5 May 2026 10:22:26 +0100 Subject: [PATCH 1/2] VED-1161: Enable S3 access logging for various resources and update configurations --- infrastructure/instance/.terraform.lock.hcl | 58 +++++++++---------- infrastructure/instance/endpoints.tf | 21 +++---- .../preprod/int-blue/variables.tfvars | 2 + .../preprod/int-green/variables.tfvars | 2 + .../instance/modules/api_gateway/mtls_cert.tf | 7 +++ .../instance/modules/api_gateway/variables.tf | 4 ++ .../instance/modules/splunk/backup.tf | 7 +++ .../instance/modules/splunk/variables.tf | 4 ++ infrastructure/instance/s3_access_logging.tf | 32 ++++++++++ infrastructure/instance/splunk.tf | 11 ++-- infrastructure/instance/variables.tf | 33 +++++++---- 11 files changed, 127 insertions(+), 54 deletions(-) create mode 100644 infrastructure/instance/s3_access_logging.tf diff --git a/infrastructure/instance/.terraform.lock.hcl b/infrastructure/instance/.terraform.lock.hcl index 9ec9f03ab7..6ba98cd431 100644 --- a/infrastructure/instance/.terraform.lock.hcl +++ b/infrastructure/instance/.terraform.lock.hcl @@ -2,25 +2,25 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "6.37.0" + version = "6.43.0" constraints = ">= 6.0.0, ~> 6.0, >= 6.28.0" hashes = [ - "h1:OdQC/z+3ReUAPhlXCJIJYUZlSQ3b96TWfFK3lCT5zKU=", - "zh:0427fadb719ed5a32feb09f047539d2348e659056f3b8a8589d34d8f0a95be7a", - "zh:3891c670674aba2125a7ac6d4348cde43646b1b46ce6f829e6f4724091bc0dcd", - "zh:632cb24b7b5790b730b33bcbe9f1a7b75f2644fb52f9d6aaafb0249c9e7601d2", - "zh:6e96ed1f824c2efa9de5b7c22ab3715624ba34c28564a06e9a15e71bc3d3a30b", - "zh:7b8fd86907b659bc45f4a3f42c3c0ccc66925a74e265b01e9e66242c0b2cafef", - "zh:81f9a587deddef4dfcc2101c54ec28a3a554056837f68ebb920c83fe8327b16f", + "h1:/A3VpeGOhvutRSlGACfUKeBMFZa3CTSLIqvT+XH0364=", + "zh:0fe91026ce8c5178781de6773531dcfcf5280ee139059dc5a0c046f1532cf389", + "zh:114001f94c38db8702210eda643ec627fa1929a88f774e17db30bc172df6759e", + "zh:1fc668d57c7edd81c06f5705b03517393444fe4988a68a3fd90b5b21fed64a55", + "zh:2bc0b4d5b706c3bbe7824bcf410942ee631d3423c23f935a51129832d81e1e17", + "zh:3f27e2befae3393df2ba423abba7f64c774d8aa6e0de20d00b35d7cca6f47d65", + "zh:410bfc19e1f38b7caabc5e1b9cd2de196bdcaa02c840372be26734775ee0214c", + "zh:417181a86499386ffbf4d023b9c5219a0d322751513806e977f146f170f0aebc", + "zh:6764d31dbae9656b698a3b9d4a44e4267210375ff9bec3e8716bac4450a06f0d", + "zh:86401475746c94b12e90b065b76a77c635a84d9cbfc57eace65131c780bd34c6", + "zh:98388ac853abbcb18fdf578c18203f485479610d28329c21deefc573976e4b1d", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:a9a38a67cb98d690fec951ec3e133b6836279629db2ed3a0ebf97a5bea58674f", - "zh:b18f60d62e4bd4d466077e09c39259d1a85355f0f00b801fe8aedbc50193d357", - "zh:b7a51bc0faf60d17043b4df1d1b7bb55129eaa4bdeb65ff55f5b00b9b8fee9f7", - "zh:c28c42f91ca3a6b65b3fd3ed6e891fc0fc28d0cb5ab65dea65eda8eec5cea5f3", - "zh:d895ddc04280ed26b6ca64ca05b78caaa7b72c8e167af4093545efbc608d5482", - "zh:f4a56f5157009ef160fbd79105078fe675df479cb73c1b7e1fea2741403a0b67", - "zh:f547d6ca371b96fec97b972fc0c93bcfc23d58e34a9da215b94e9d2aa170fb77", - "zh:f7b0a3cd4adadd3f4b9609a54e651ed5eafa22c196ab229042fc1d0aa0ab8f3a", + "zh:afb84c77c569e9979b8287cde33543cff992ca17e62ebe3f2b4a8e69884dbdc5", + "zh:c54c64dba350e6856fb8f2813b81d20286a532b02bad5f67da35135c02594407", + "zh:d1a46adf1c8f5bf42a1886b06fd25d86981236c933165f0fbc07e4330b77d8d1", + "zh:e719e227a676588cdaa5a3e7e3e6b10da26830a566730e8bce2127eec1780f40", ] } @@ -46,22 +46,22 @@ provider "registry.terraform.io/hashicorp/external" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.7.0" + version = "2.8.0" constraints = ">= 1.0.0" hashes = [ - "h1:sSwlfp2etjCaE9hIF7bJBDjRIhDCVFglEOVyiCI7vgs=", - "zh:261fec71bca13e0a7812dc0d8ae9af2b4326b24d9b2e9beab3d2400fab5c5f9a", - "zh:308da3b5376a9ede815042deec5af1050ec96a5a5410a2206ae847d82070a23e", - "zh:3d056924c420464dc8aba10e1915956b2e5c4d55b11ffff79aa8be563fbfe298", - "zh:643256547b155459c45e0a3e8aab0570db59923c68daf2086be63c444c8c445b", + "h1:3jWHVwO5QUIS9V1NsK10ZzdpkK2ABuB4G+UIWrVeGp4=", + "zh:05f18164beab4a84753e5fedf463771ee0c6eca8e90346b8766f1e1c186dec1e", + "zh:563a0702e3711e25ba8930120899b681378b50cbb957fd204b37745c7c9b5f40", + "zh:5b56ab2ed70ed92721febb4a070af0837f1084c44825c18e4b95f7efb1d45d26", + "zh:6cbedc09b67a5cdb9501ff1b18a315fa46a38e0530424cab1c7f4b3acc75f489", + "zh:71b3bd50f89fb385a42a436ba2ce2b8e00f9de53535ce956deff1477b0b117dc", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7aa4d0b853f84205e8cf79f30c9b2c562afbfa63592f7231b6637e5d7a6b5b27", - "zh:7dc251bbc487d58a6ab7f5b07ec9edc630edb45d89b761dba28e0e2ba6b1c11f", - "zh:7ee0ca546cd065030039168d780a15cbbf1765a4c70cd56d394734ab112c93da", - "zh:b1d5d80abb1906e6c6b3685a52a0192b4ca6525fe090881c64ec6f67794b1300", - "zh:d81ea9856d61db3148a4fc6c375bf387a721d78fc1fea7a8823a027272a47a78", - "zh:df0a1f0afc947b8bfc88617c1ad07a689ce3bd1a29fd97318392e6bdd32b230b", - "zh:dfbcad800240e0c68c43e0866f2a751cff09777375ec701918881acf67a268da", + "zh:9d45ac0a00b85cabdd398b859349d17f124c598b6e6bf272f1bb01321ce708a8", + "zh:a453efe8641a8f31fe806b597bf2b34d7b78b971a8e3919061ea89d61fda7b8d", + "zh:ac692bacb8c3dca8b5b37e5383168aca1f87d3cd7b40615efd300defb76494f5", + "zh:bda9e90c8547d90c9c573206985c5675cc1406047605af037a5069942c3c5966", + "zh:c30a1967de040d00f5038086dd53cdbfb78cc05d1dbc75037410f011bf2a20d8", + "zh:c80bbd1c3f56b3c836d80cf93ac0e8809305c2642f0c98b54bf5d05d3b12718c", ] } diff --git a/infrastructure/instance/endpoints.tf b/infrastructure/instance/endpoints.tf index 35b80b8ea8..822bfd6590 100644 --- a/infrastructure/instance/endpoints.tf +++ b/infrastructure/instance/endpoints.tf @@ -111,16 +111,17 @@ output "oas" { module "api_gateway" { source = "./modules/api_gateway" - prefix = local.prefix - short_prefix = local.short_prefix - zone_id = data.aws_route53_zone.project_zone.zone_id - api_domain_name = local.service_domain_name - environment = var.environment - sub_environment = var.sub_environment - oas = local.oas - aws_region = var.aws_region - immunisation_account_id = var.immunisation_account_id - csoc_account_id = var.csoc_account_id + prefix = local.prefix + short_prefix = local.short_prefix + zone_id = data.aws_route53_zone.project_zone.zone_id + api_domain_name = local.service_domain_name + environment = var.environment + sub_environment = var.sub_environment + oas = local.oas + aws_region = var.aws_region + immunisation_account_id = var.immunisation_account_id + csoc_account_id = var.csoc_account_id + access_log_target_bucket = var.enable_s3_access_logging ? local.s3_access_log_bucket_name : null } resource "aws_lambda_permission" "api_gw" { diff --git a/infrastructure/instance/environments/preprod/int-blue/variables.tfvars b/infrastructure/instance/environments/preprod/int-blue/variables.tfvars index 0b3107be75..4e35674ad1 100644 --- a/infrastructure/instance/environments/preprod/int-blue/variables.tfvars +++ b/infrastructure/instance/environments/preprod/int-blue/variables.tfvars @@ -10,3 +10,5 @@ mesh_no_invocation_period_seconds = 259200 create_mesh_processor = true has_sub_environment_scope = false dynamodb_point_in_time_recovery_enabled = true +enable_s3_access_logging = true +s3_access_log_bucket_name = "immunisation-preprod-s3-access-logs" diff --git a/infrastructure/instance/environments/preprod/int-green/variables.tfvars b/infrastructure/instance/environments/preprod/int-green/variables.tfvars index 0b3107be75..4e35674ad1 100644 --- a/infrastructure/instance/environments/preprod/int-green/variables.tfvars +++ b/infrastructure/instance/environments/preprod/int-green/variables.tfvars @@ -10,3 +10,5 @@ mesh_no_invocation_period_seconds = 259200 create_mesh_processor = true has_sub_environment_scope = false dynamodb_point_in_time_recovery_enabled = true +enable_s3_access_logging = true +s3_access_log_bucket_name = "immunisation-preprod-s3-access-logs" diff --git a/infrastructure/instance/modules/api_gateway/mtls_cert.tf b/infrastructure/instance/modules/api_gateway/mtls_cert.tf index 054517c104..7c3648d351 100644 --- a/infrastructure/instance/modules/api_gateway/mtls_cert.tf +++ b/infrastructure/instance/modules/api_gateway/mtls_cert.tf @@ -21,6 +21,13 @@ resource "aws_s3_bucket" "truststore_bucket" { force_destroy = true } +resource "aws_s3_bucket_logging" "truststore_bucket" { + count = var.access_log_target_bucket == null ? 0 : 1 + bucket = aws_s3_bucket.truststore_bucket.bucket + target_bucket = var.access_log_target_bucket + target_prefix = "${aws_s3_bucket.truststore_bucket.bucket}/" +} + resource "aws_s3_bucket_versioning" "truststore_bucket" { bucket = aws_s3_bucket.truststore_bucket.bucket versioning_configuration { diff --git a/infrastructure/instance/modules/api_gateway/variables.tf b/infrastructure/instance/modules/api_gateway/variables.tf index aaa0e5d845..8a55588a2b 100644 --- a/infrastructure/instance/modules/api_gateway/variables.tf +++ b/infrastructure/instance/modules/api_gateway/variables.tf @@ -16,3 +16,7 @@ variable "aws_region" { } variable "immunisation_account_id" {} variable "csoc_account_id" {} +variable "access_log_target_bucket" { + type = string + default = null +} diff --git a/infrastructure/instance/modules/splunk/backup.tf b/infrastructure/instance/modules/splunk/backup.tf index 77450ce335..4e60a47384 100644 --- a/infrastructure/instance/modules/splunk/backup.tf +++ b/infrastructure/instance/modules/splunk/backup.tf @@ -3,3 +3,10 @@ resource "aws_s3_bucket" "failed_logs_backup" { // To facilitate deletion of non empty busckets force_destroy = var.force_destroy } + +resource "aws_s3_bucket_logging" "failed_logs_backup" { + count = var.access_log_target_bucket == null ? 0 : 1 + bucket = aws_s3_bucket.failed_logs_backup.bucket + target_bucket = var.access_log_target_bucket + target_prefix = "${aws_s3_bucket.failed_logs_backup.bucket}/" +} diff --git a/infrastructure/instance/modules/splunk/variables.tf b/infrastructure/instance/modules/splunk/variables.tf index 7f4ac4f86a..238c8121a7 100644 --- a/infrastructure/instance/modules/splunk/variables.tf +++ b/infrastructure/instance/modules/splunk/variables.tf @@ -5,3 +5,7 @@ locals { variable "splunk_endpoint" {} variable "hec_token" {} variable "force_destroy" {} +variable "access_log_target_bucket" { + type = string + default = null +} diff --git a/infrastructure/instance/s3_access_logging.tf b/infrastructure/instance/s3_access_logging.tf new file mode 100644 index 0000000000..0052165477 --- /dev/null +++ b/infrastructure/instance/s3_access_logging.tf @@ -0,0 +1,32 @@ +data "aws_s3_bucket" "existing_s3_access_log_bucket" { + count = var.enable_s3_access_logging ? 1 : 0 + bucket = local.s3_access_log_bucket_name +} + +resource "aws_s3_bucket_logging" "batch_data_source_bucket" { + count = var.enable_s3_access_logging ? 1 : 0 + bucket = aws_s3_bucket.batch_data_source_bucket.bucket + target_bucket = data.aws_s3_bucket.existing_s3_access_log_bucket[0].bucket + target_prefix = "${aws_s3_bucket.batch_data_source_bucket.bucket}/" +} + +resource "aws_s3_bucket_logging" "batch_data_destination_bucket" { + count = var.enable_s3_access_logging ? 1 : 0 + bucket = aws_s3_bucket.batch_data_destination_bucket.bucket + target_bucket = data.aws_s3_bucket.existing_s3_access_log_bucket[0].bucket + target_prefix = "${aws_s3_bucket.batch_data_destination_bucket.bucket}/" +} + +resource "aws_s3_bucket_logging" "batch_config_bucket" { + count = var.enable_s3_access_logging ? 1 : 0 + bucket = aws_s3_bucket.batch_config_bucket.bucket + target_bucket = data.aws_s3_bucket.existing_s3_access_log_bucket[0].bucket + target_prefix = "${aws_s3_bucket.batch_config_bucket.bucket}/" +} + +resource "aws_s3_bucket_logging" "account_batch_data_source_bucket" { + count = var.enable_s3_access_logging && !var.has_sub_environment_scope ? 1 : 0 + bucket = "immunisation-batch-${local.resource_scope}-data-sources" + target_bucket = data.aws_s3_bucket.existing_s3_access_log_bucket[0].bucket + target_prefix = "immunisation-batch-${local.resource_scope}-data-sources/" +} diff --git a/infrastructure/instance/splunk.tf b/infrastructure/instance/splunk.tf index 21ce6d7510..730b70dac7 100644 --- a/infrastructure/instance/splunk.tf +++ b/infrastructure/instance/splunk.tf @@ -6,9 +6,10 @@ data "aws_secretsmanager_secret_version" "splunk_token_id" { } module "splunk" { - source = "./modules/splunk" - prefix = local.prefix - splunk_endpoint = "https://firehose.inputs.splunk.aws.digital.nhs.uk/services/collector/event" - hec_token = data.aws_secretsmanager_secret_version.splunk_token_id.secret_string - force_destroy = local.is_temp + source = "./modules/splunk" + prefix = local.prefix + splunk_endpoint = "https://firehose.inputs.splunk.aws.digital.nhs.uk/services/collector/event" + hec_token = data.aws_secretsmanager_secret_version.splunk_token_id.secret_string + force_destroy = local.is_temp + access_log_target_bucket = var.enable_s3_access_logging ? local.s3_access_log_bucket_name : null } diff --git a/infrastructure/instance/variables.tf b/infrastructure/instance/variables.tf index 39e7b7e07a..cb7c34de0e 100644 --- a/infrastructure/instance/variables.tf +++ b/infrastructure/instance/variables.tf @@ -103,17 +103,30 @@ variable "dynamodb_point_in_time_recovery_enabled" { default = false } +variable "s3_access_log_bucket_name" { + description = "Destination bucket used for S3 server access logs" + type = string + default = "" +} + +variable "enable_s3_access_logging" { + description = "When true, manage S3 server access logging resources in this stack" + type = bool + default = false +} + locals { - prefix = "${var.project_name}-${var.service}-${var.sub_environment}" - short_prefix = "${var.project_short_name}-${var.sub_environment}" - batch_prefix = "immunisation-batch-${var.sub_environment}" - root_domain_name = "${var.environment}.vds.platform.nhs.uk" - project_domain_name = "imms.${local.root_domain_name}" - service_domain_name = "${var.sub_environment}.${local.project_domain_name}" - config_bucket_arn = aws_s3_bucket.batch_config_bucket.arn - config_bucket_name = aws_s3_bucket.batch_config_bucket.bucket - is_temp = length(regexall("[a-z]{2,4}-?[0-9]+", var.sub_environment)) > 0 - resource_scope = var.has_sub_environment_scope ? var.sub_environment : var.environment + prefix = "${var.project_name}-${var.service}-${var.sub_environment}" + short_prefix = "${var.project_short_name}-${var.sub_environment}" + batch_prefix = "immunisation-batch-${var.sub_environment}" + root_domain_name = "${var.environment}.vds.platform.nhs.uk" + project_domain_name = "imms.${local.root_domain_name}" + service_domain_name = "${var.sub_environment}.${local.project_domain_name}" + config_bucket_arn = aws_s3_bucket.batch_config_bucket.arn + config_bucket_name = aws_s3_bucket.batch_config_bucket.bucket + is_temp = length(regexall("[a-z]{2,4}-?[0-9]+", var.sub_environment)) > 0 + resource_scope = var.has_sub_environment_scope ? var.sub_environment : var.environment + s3_access_log_bucket_name = var.s3_access_log_bucket_name != "" ? var.s3_access_log_bucket_name : "immunisation-${var.environment}-s3-access-logs" # Public subnet - The subnet has a direct route to an internet gateway. Resources in a public subnet can access the public internet. # public_subnet_ids = [for k, v in data.aws_route.internet_traffic_route_by_subnet : k if length(v.gateway_id) > 0] # Private subnet - The subnet does not have a direct route to an internet gateway. Resources in a private subnet require a NAT device to access the public internet. From 6406a4ff7bd65cf0ba213f95d81ec5d404497cf0 Mon Sep 17 00:00:00 2001 From: amarauzoma Date: Tue, 5 May 2026 10:33:10 +0100 Subject: [PATCH 2/2] VED-1161: prepare terraform merge resolution --- .../instance/modules/splunk/backup.tf | 30 +++++ infrastructure/instance/variables.tf | 121 ++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/infrastructure/instance/modules/splunk/backup.tf b/infrastructure/instance/modules/splunk/backup.tf index 4e60a47384..698eee5509 100644 --- a/infrastructure/instance/modules/splunk/backup.tf +++ b/infrastructure/instance/modules/splunk/backup.tf @@ -4,6 +4,36 @@ resource "aws_s3_bucket" "failed_logs_backup" { force_destroy = var.force_destroy } +data "aws_iam_policy_document" "failed_logs_backup_https_only" { + statement { + sid = "HTTPSOnly" + effect = "Deny" + + principals { + type = "AWS" + identifiers = ["*"] + } + + actions = ["s3:*"] + + resources = [ + aws_s3_bucket.failed_logs_backup.arn, + "${aws_s3_bucket.failed_logs_backup.arn}/*", + ] + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} + +resource "aws_s3_bucket_policy" "failed_logs_backup_https_only" { + bucket = aws_s3_bucket.failed_logs_backup.id + policy = data.aws_iam_policy_document.failed_logs_backup_https_only.json +} + resource "aws_s3_bucket_logging" "failed_logs_backup" { count = var.access_log_target_bucket == null ? 0 : 1 bucket = aws_s3_bucket.failed_logs_backup.bucket diff --git a/infrastructure/instance/variables.tf b/infrastructure/instance/variables.tf index cb7c34de0e..9aa6e05a1d 100644 --- a/infrastructure/instance/variables.tf +++ b/infrastructure/instance/variables.tf @@ -103,6 +103,127 @@ variable "dynamodb_point_in_time_recovery_enabled" { default = false } +variable "recordprocessor_image_uri" { + description = "Immutable URI of the recordprocessor (batch processor) container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.recordprocessor_image_uri) != "" + error_message = "recordprocessor_image_uri must be provided." + } +} + +variable "backend_image_uri" { + description = "Immutable URI of the backend Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.backend_image_uri) != "" + error_message = "backend_image_uri must be provided." + } +} + +variable "ack_backend_image_uri" { + description = "Immutable URI of the ack backend Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.ack_backend_image_uri) != "" + error_message = "ack_backend_image_uri must be provided." + } +} + +variable "batch_processor_filter_image_uri" { + description = "Immutable URI of the batch processor filter Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.batch_processor_filter_image_uri) != "" + error_message = "batch_processor_filter_image_uri must be provided." + } +} + +variable "delta_backend_image_uri" { + description = "Immutable URI of the delta backend Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.delta_backend_image_uri) != "" + error_message = "delta_backend_image_uri must be provided." + } +} + +variable "filenameprocessor_image_uri" { + description = "Immutable URI of the filenameprocessor Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.filenameprocessor_image_uri) != "" + error_message = "filenameprocessor_image_uri must be provided." + } +} + +variable "id_sync_image_uri" { + description = "Immutable URI of the id sync Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.id_sync_image_uri) != "" + error_message = "id_sync_image_uri must be provided." + } +} + +variable "mesh_processor_image_uri" { + description = "Immutable URI of the mesh processor Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.mesh_processor_image_uri) != "" + error_message = "mesh_processor_image_uri must be provided." + } +} + +variable "mns_publisher_image_uri" { + description = "Immutable URI of the MNS publisher Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.mns_publisher_image_uri) != "" + error_message = "mns_publisher_image_uri must be provided." + } +} + +variable "recordforwarder_image_uri" { + description = "Immutable URI of the recordforwarder Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.recordforwarder_image_uri) != "" + error_message = "recordforwarder_image_uri must be provided." + } +} + +variable "redis_sync_image_uri" { + description = "Immutable URI of the redis sync Lambda container image in ECR. Must be supplied by CI/CD." + type = string + default = "" + + validation { + condition = trimspace(var.redis_sync_image_uri) != "" + error_message = "redis_sync_image_uri must be provided." + } +} + variable "s3_access_log_bucket_name" { description = "Destination bucket used for S3 server access logs" type = string