From e533fc7570e4afe6ffd2b41fe10e1228a660e90b Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Tue, 31 Mar 2026 09:53:35 +0100 Subject: [PATCH 1/7] chore: otel for lambdas --- infra/modules/aws/_shared/lambda/data.tf | 17 +++++++++++++++++ infra/modules/aws/_shared/lambda/locals.tf | 2 +- infra/modules/aws/_shared/lambda/main.tf | 19 ++++++++++++++++++- infra/modules/aws/_shared/lambda/variables.tf | 5 +++++ infra/root.hcl | 3 ++- 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/infra/modules/aws/_shared/lambda/data.tf b/infra/modules/aws/_shared/lambda/data.tf index 1ef311c..a0d22f6 100644 --- a/infra/modules/aws/_shared/lambda/data.tf +++ b/infra/modules/aws/_shared/lambda/data.tf @@ -1,3 +1,20 @@ +data "aws_region" "current" {} + +data "aws_lambda_layer_version" "otel" { + layer_name = "arn:aws:lambda:${data.aws_region.current.name}:${var.aws_lambda_layers_account_id}:layer:aws-otel-python-amd64-ver-1-30-0" +} + +data "aws_iam_policy_document" "lambda_xray" { + statement { + effect = "Allow" + actions = [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ] + resources = ["*"] + } +} + data "aws_s3_bucket" "code_bucket" { bucket = var.code_bucket } diff --git a/infra/modules/aws/_shared/lambda/locals.tf b/infra/modules/aws/_shared/lambda/locals.tf index 1a7ae80..36a8d05 100644 --- a/infra/modules/aws/_shared/lambda/locals.tf +++ b/infra/modules/aws/_shared/lambda/locals.tf @@ -1,5 +1,5 @@ locals { - lambda_runtime = "python3.12" + lambda_runtime = "python3.12" lambda_handler = "lambda_handler.lambda_handler" compute_platform = "Lambda" diff --git a/infra/modules/aws/_shared/lambda/main.tf b/infra/modules/aws/_shared/lambda/main.tf index 52dd042..2b1c06d 100644 --- a/infra/modules/aws/_shared/lambda/main.tf +++ b/infra/modules/aws/_shared/lambda/main.tf @@ -30,11 +30,22 @@ resource "aws_s3_object" "bootstrap_lambda_zip" { content_type = "application/zip" } +resource "aws_iam_policy" "lambda_xray" { + name = "${local.lambda_name}-xray" + policy = data.aws_iam_policy_document.lambda_xray.json +} + +resource "aws_iam_role_policy_attachment" "lambda_xray" { + role = aws_iam_role.iam_for_lambda.name + policy_arn = aws_iam_policy.lambda_xray.arn +} + resource "aws_lambda_function" "lambda" { function_name = local.lambda_name role = aws_iam_role.iam_for_lambda.arn handler = local.lambda_handler runtime = local.lambda_runtime + layers = [data.aws_lambda_layer_version.otel.arn] reserved_concurrent_executions = local.pc_reserved_count @@ -44,8 +55,14 @@ resource "aws_lambda_function" "lambda" { # publish ONE immutable version so we can create an alias publish = true + tracing_config { + mode = "Active" + } + environment { - variables = var.environment_variables + variables = merge(var.environment_variables, { + AWS_LAMBDA_EXEC_WRAPPER = "/opt/otel-instrument" + }) } # tags for identifying the code deploy app and its deployment config. Used in CI/CD pipelines. diff --git a/infra/modules/aws/_shared/lambda/variables.tf b/infra/modules/aws/_shared/lambda/variables.tf index 83ed6dd..d3fe9bd 100644 --- a/infra/modules/aws/_shared/lambda/variables.tf +++ b/infra/modules/aws/_shared/lambda/variables.tf @@ -1,4 +1,9 @@ ### start of static vars set in root.hcl ### +variable "aws_lambda_layers_account_id" { + type = string + description = "AWS account ID that publishes the ADOT Lambda layers" +} + variable "project_name" { type = string description = "Project name used in naming resources" diff --git a/infra/root.hcl b/infra/root.hcl index 99f15a2..b54f21e 100644 --- a/infra/root.hcl +++ b/infra/root.hcl @@ -88,7 +88,8 @@ inputs = merge( local.global_vars.inputs, local.environment_vars.inputs, { - aws_account_id = local.aws_account_id + aws_account_id = local.aws_account_id + aws_lambda_layers_account_id = "901920570463" aws_region = local.aws_region project_name = local.project_name environment = local.environment From 84500d0e84109fd9a3f1bf992ed8df8b3e400faa Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Tue, 31 Mar 2026 10:31:21 +0100 Subject: [PATCH 2/7] chore: set otel sampling rates --- infra/live/prod/environment_vars.hcl | 1 + infra/modules/aws/_shared/lambda/main.tf | 2 ++ infra/modules/aws/_shared/lambda/variables.tf | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/infra/live/prod/environment_vars.hcl b/infra/live/prod/environment_vars.hcl index dfaeeca..679e89f 100644 --- a/infra/live/prod/environment_vars.hcl +++ b/infra/live/prod/environment_vars.hcl @@ -6,4 +6,5 @@ locals { inputs = { log_retention_days = local.log_retention_days deploy_branches = local.deploy_branches + otel_sample_rate = 0.1 # 10% of traces sampled } diff --git a/infra/modules/aws/_shared/lambda/main.tf b/infra/modules/aws/_shared/lambda/main.tf index 2b1c06d..480534a 100644 --- a/infra/modules/aws/_shared/lambda/main.tf +++ b/infra/modules/aws/_shared/lambda/main.tf @@ -62,6 +62,8 @@ resource "aws_lambda_function" "lambda" { environment { variables = merge(var.environment_variables, { AWS_LAMBDA_EXEC_WRAPPER = "/opt/otel-instrument" + OTEL_TRACES_SAMPLER = "parentbased_traceidratio" + OTEL_TRACES_SAMPLER_ARG = tostring(var.otel_sample_rate) }) } diff --git a/infra/modules/aws/_shared/lambda/variables.tf b/infra/modules/aws/_shared/lambda/variables.tf index d3fe9bd..09aef59 100644 --- a/infra/modules/aws/_shared/lambda/variables.tf +++ b/infra/modules/aws/_shared/lambda/variables.tf @@ -4,6 +4,12 @@ variable "aws_lambda_layers_account_id" { description = "AWS account ID that publishes the ADOT Lambda layers" } +variable "otel_sample_rate" { + type = number + description = "OpenTelemetry trace sampling rate — 0.0 = no traces, 1.0 = 100% of traces sampled" + default = 1.0 # 100% +} + variable "project_name" { type = string description = "Project name used in naming resources" From c036b72a222c53bb79047aed66e41bba6432ea98 Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Tue, 31 Mar 2026 10:33:25 +0100 Subject: [PATCH 3/7] fix: fmt --- infra/modules/aws/_shared/lambda/locals.tf | 2 +- infra/root.hcl | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/infra/modules/aws/_shared/lambda/locals.tf b/infra/modules/aws/_shared/lambda/locals.tf index 36a8d05..1a7ae80 100644 --- a/infra/modules/aws/_shared/lambda/locals.tf +++ b/infra/modules/aws/_shared/lambda/locals.tf @@ -1,5 +1,5 @@ locals { - lambda_runtime = "python3.12" + lambda_runtime = "python3.12" lambda_handler = "lambda_handler.lambda_handler" compute_platform = "Lambda" diff --git a/infra/root.hcl b/infra/root.hcl index b54f21e..854cd80 100644 --- a/infra/root.hcl +++ b/infra/root.hcl @@ -88,16 +88,16 @@ inputs = merge( local.global_vars.inputs, local.environment_vars.inputs, { - aws_account_id = local.aws_account_id - aws_lambda_layers_account_id = "901920570463" - aws_region = local.aws_region - project_name = local.project_name - environment = local.environment - github_repo = local.github_repo - deploy_role_name = local.deploy_role_name - deploy_role_arn = local.deploy_role_arn - state_bucket = local.state_bucket - state_lock_table = local.state_lock_table - code_bucket = local.code_bucket + aws_account_id = local.aws_account_id + aws_lambda_layers_account_id = "901920570463" + aws_region = local.aws_region + project_name = local.project_name + environment = local.environment + github_repo = local.github_repo + deploy_role_name = local.deploy_role_name + deploy_role_arn = local.deploy_role_arn + state_bucket = local.state_bucket + state_lock_table = local.state_lock_table + code_bucket = local.code_bucket } ) From 0b125f0cc7fcaa01f69b85ed552606cb7d6eb009 Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Tue, 31 Mar 2026 10:35:30 +0100 Subject: [PATCH 4/7] chore: mv aws_lambda_layers_account_id --- infra/root.hcl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infra/root.hcl b/infra/root.hcl index 854cd80..24b3515 100644 --- a/infra/root.hcl +++ b/infra/root.hcl @@ -16,6 +16,8 @@ locals { project_name = element(split("/", local.github_repo), 1) + aws_lambda_layers_account_id = "901920570463" + aws_region = local.global_vars.inputs.aws_region base_reference = "${local.aws_account_id}-${local.aws_region}-${local.project_name}" deploy_role_name = "${local.project_name}-${local.environment}-github-oidc-role" @@ -89,7 +91,7 @@ inputs = merge( local.environment_vars.inputs, { aws_account_id = local.aws_account_id - aws_lambda_layers_account_id = "901920570463" + aws_lambda_layers_account_id = local.aws_lambda_layers_account_id aws_region = local.aws_region project_name = local.project_name environment = local.environment From 99089125cbcddd9743c08b29ebf07f7df7bd47e4 Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Tue, 31 Mar 2026 10:40:24 +0100 Subject: [PATCH 5/7] fix: pass in aws_lambda_layers_account_id var --- infra/modules/aws/api/main.tf | 8 +++++--- infra/modules/aws/api/variables.tf | 11 +++++++++++ infra/modules/aws/lambda_worker/main.tf | 8 +++++--- infra/modules/aws/lambda_worker/variables.tf | 11 +++++++++++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/infra/modules/aws/api/main.tf b/infra/modules/aws/api/main.tf index d8c3e00..604c883 100644 --- a/infra/modules/aws/api/main.tf +++ b/infra/modules/aws/api/main.tf @@ -1,9 +1,11 @@ module "lambda_api" { source = "../_shared/lambda" - project_name = var.project_name - environment = var.environment - code_bucket = var.code_bucket + project_name = var.project_name + environment = var.environment + code_bucket = var.code_bucket + aws_lambda_layers_account_id = var.aws_lambda_layers_account_id + otel_sample_rate = var.otel_sample_rate lambda_name = local.lambda_name diff --git a/infra/modules/aws/api/variables.tf b/infra/modules/aws/api/variables.tf index c34d4f9..bc3f1f6 100644 --- a/infra/modules/aws/api/variables.tf +++ b/infra/modules/aws/api/variables.tf @@ -13,6 +13,17 @@ variable "code_bucket" { type = string description = "Bucket where deployable code artifacts are uploaded" } + +variable "aws_lambda_layers_account_id" { + type = string + description = "AWS account ID that publishes the ADOT Lambda layers" +} + +variable "otel_sample_rate" { + type = number + description = "OpenTelemetry trace sampling rate — 0.0 = no traces, 1.0 = 100% of traces sampled" + default = 1.0 # 100% +} ### end of static vars set in root.hcl ### variable "deployment_config" { diff --git a/infra/modules/aws/lambda_worker/main.tf b/infra/modules/aws/lambda_worker/main.tf index ce45336..1061d3e 100644 --- a/infra/modules/aws/lambda_worker/main.tf +++ b/infra/modules/aws/lambda_worker/main.tf @@ -1,9 +1,11 @@ module "lambda_worker" { source = "../_shared/lambda" - project_name = var.project_name - environment = var.environment - code_bucket = var.code_bucket + project_name = var.project_name + environment = var.environment + code_bucket = var.code_bucket + aws_lambda_layers_account_id = var.aws_lambda_layers_account_id + otel_sample_rate = var.otel_sample_rate lambda_name = local.lambda_name diff --git a/infra/modules/aws/lambda_worker/variables.tf b/infra/modules/aws/lambda_worker/variables.tf index ba56938..c261cb7 100644 --- a/infra/modules/aws/lambda_worker/variables.tf +++ b/infra/modules/aws/lambda_worker/variables.tf @@ -13,6 +13,17 @@ variable "code_bucket" { type = string description = "Bucket where deployable code artifacts are uploaded" } + +variable "aws_lambda_layers_account_id" { + type = string + description = "AWS account ID that publishes the ADOT Lambda layers" +} + +variable "otel_sample_rate" { + type = number + description = "OpenTelemetry trace sampling rate — 0.0 = no traces, 1.0 = 100% of traces sampled" + default = 1.0 # 100% +} ### end of static vars set in root.hcl ### variable "sqs_queue_name" { From e0c53f5bed0f4e8412c33ba376c2794da1724171 Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Tue, 31 Mar 2026 10:46:51 +0100 Subject: [PATCH 6/7] fix: use arn string for otel layer --- infra/modules/aws/_shared/lambda/data.tf | 4 ---- infra/modules/aws/_shared/lambda/locals.tf | 4 +++- infra/modules/aws/_shared/lambda/main.tf | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/infra/modules/aws/_shared/lambda/data.tf b/infra/modules/aws/_shared/lambda/data.tf index a0d22f6..cbfc8a8 100644 --- a/infra/modules/aws/_shared/lambda/data.tf +++ b/infra/modules/aws/_shared/lambda/data.tf @@ -1,9 +1,5 @@ data "aws_region" "current" {} -data "aws_lambda_layer_version" "otel" { - layer_name = "arn:aws:lambda:${data.aws_region.current.name}:${var.aws_lambda_layers_account_id}:layer:aws-otel-python-amd64-ver-1-30-0" -} - data "aws_iam_policy_document" "lambda_xray" { statement { effect = "Allow" diff --git a/infra/modules/aws/_shared/lambda/locals.tf b/infra/modules/aws/_shared/lambda/locals.tf index 1a7ae80..10e59d3 100644 --- a/infra/modules/aws/_shared/lambda/locals.tf +++ b/infra/modules/aws/_shared/lambda/locals.tf @@ -1,5 +1,7 @@ locals { - lambda_runtime = "python3.12" + lambda_runtime = "python3.12" + # Update the layer name and version when upgrading ADOT: https://aws-otel.github.io/docs/getting-started/lambda + otel_layer_arn = "arn:aws:lambda:${data.aws_region.current.name}:${var.aws_lambda_layers_account_id}:layer:aws-otel-python-amd64-ver-1-30-0:1" lambda_handler = "lambda_handler.lambda_handler" compute_platform = "Lambda" diff --git a/infra/modules/aws/_shared/lambda/main.tf b/infra/modules/aws/_shared/lambda/main.tf index 480534a..7b95c44 100644 --- a/infra/modules/aws/_shared/lambda/main.tf +++ b/infra/modules/aws/_shared/lambda/main.tf @@ -45,7 +45,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.iam_for_lambda.arn handler = local.lambda_handler runtime = local.lambda_runtime - layers = [data.aws_lambda_layer_version.otel.arn] + layers = [local.otel_layer_arn] reserved_concurrent_executions = local.pc_reserved_count From 5e58912f27cc138848dba2fb6d53036be5fd8eb7 Mon Sep 17 00:00:00 2001 From: chrispsheehan Date: Tue, 31 Mar 2026 11:50:37 +0100 Subject: [PATCH 7/7] fix: layers not needed --- infra/live/global_vars.hcl | 3 ++- infra/modules/aws/_shared/lambda/data.tf | 2 -- infra/modules/aws/_shared/lambda/locals.tf | 4 +--- infra/modules/aws/_shared/lambda/main.tf | 11 ++++----- infra/modules/aws/_shared/lambda/variables.tf | 5 ---- infra/modules/aws/api/main.tf | 9 ++++---- infra/modules/aws/api/variables.tf | 5 ---- infra/modules/aws/lambda_worker/main.tf | 9 ++++---- infra/modules/aws/lambda_worker/variables.tf | 5 ---- infra/root.hcl | 23 ++++++++----------- 10 files changed, 25 insertions(+), 51 deletions(-) diff --git a/infra/live/global_vars.hcl b/infra/live/global_vars.hcl index a863909..1f9e675 100644 --- a/infra/live/global_vars.hcl +++ b/infra/live/global_vars.hcl @@ -10,7 +10,8 @@ locals { "application-autoscaling:*", "cloudwatch:*", "sqs:*", - "cloudfront:*" + "cloudfront:*", + "xray:*" ] } diff --git a/infra/modules/aws/_shared/lambda/data.tf b/infra/modules/aws/_shared/lambda/data.tf index cbfc8a8..1b2b71e 100644 --- a/infra/modules/aws/_shared/lambda/data.tf +++ b/infra/modules/aws/_shared/lambda/data.tf @@ -1,5 +1,3 @@ -data "aws_region" "current" {} - data "aws_iam_policy_document" "lambda_xray" { statement { effect = "Allow" diff --git a/infra/modules/aws/_shared/lambda/locals.tf b/infra/modules/aws/_shared/lambda/locals.tf index 10e59d3..1a7ae80 100644 --- a/infra/modules/aws/_shared/lambda/locals.tf +++ b/infra/modules/aws/_shared/lambda/locals.tf @@ -1,7 +1,5 @@ locals { - lambda_runtime = "python3.12" - # Update the layer name and version when upgrading ADOT: https://aws-otel.github.io/docs/getting-started/lambda - otel_layer_arn = "arn:aws:lambda:${data.aws_region.current.name}:${var.aws_lambda_layers_account_id}:layer:aws-otel-python-amd64-ver-1-30-0:1" + lambda_runtime = "python3.12" lambda_handler = "lambda_handler.lambda_handler" compute_platform = "Lambda" diff --git a/infra/modules/aws/_shared/lambda/main.tf b/infra/modules/aws/_shared/lambda/main.tf index 7b95c44..c59f5ed 100644 --- a/infra/modules/aws/_shared/lambda/main.tf +++ b/infra/modules/aws/_shared/lambda/main.tf @@ -41,12 +41,10 @@ resource "aws_iam_role_policy_attachment" "lambda_xray" { } resource "aws_lambda_function" "lambda" { - function_name = local.lambda_name - role = aws_iam_role.iam_for_lambda.arn - handler = local.lambda_handler - runtime = local.lambda_runtime - layers = [local.otel_layer_arn] - + function_name = local.lambda_name + role = aws_iam_role.iam_for_lambda.arn + handler = local.lambda_handler + runtime = local.lambda_runtime reserved_concurrent_executions = local.pc_reserved_count s3_bucket = data.aws_s3_bucket.code_bucket.bucket @@ -61,7 +59,6 @@ resource "aws_lambda_function" "lambda" { environment { variables = merge(var.environment_variables, { - AWS_LAMBDA_EXEC_WRAPPER = "/opt/otel-instrument" OTEL_TRACES_SAMPLER = "parentbased_traceidratio" OTEL_TRACES_SAMPLER_ARG = tostring(var.otel_sample_rate) }) diff --git a/infra/modules/aws/_shared/lambda/variables.tf b/infra/modules/aws/_shared/lambda/variables.tf index 09aef59..9612c53 100644 --- a/infra/modules/aws/_shared/lambda/variables.tf +++ b/infra/modules/aws/_shared/lambda/variables.tf @@ -1,9 +1,4 @@ ### start of static vars set in root.hcl ### -variable "aws_lambda_layers_account_id" { - type = string - description = "AWS account ID that publishes the ADOT Lambda layers" -} - variable "otel_sample_rate" { type = number description = "OpenTelemetry trace sampling rate — 0.0 = no traces, 1.0 = 100% of traces sampled" diff --git a/infra/modules/aws/api/main.tf b/infra/modules/aws/api/main.tf index 604c883..4714b87 100644 --- a/infra/modules/aws/api/main.tf +++ b/infra/modules/aws/api/main.tf @@ -1,11 +1,10 @@ module "lambda_api" { source = "../_shared/lambda" - project_name = var.project_name - environment = var.environment - code_bucket = var.code_bucket - aws_lambda_layers_account_id = var.aws_lambda_layers_account_id - otel_sample_rate = var.otel_sample_rate + project_name = var.project_name + environment = var.environment + code_bucket = var.code_bucket + otel_sample_rate = var.otel_sample_rate lambda_name = local.lambda_name diff --git a/infra/modules/aws/api/variables.tf b/infra/modules/aws/api/variables.tf index bc3f1f6..5542827 100644 --- a/infra/modules/aws/api/variables.tf +++ b/infra/modules/aws/api/variables.tf @@ -14,11 +14,6 @@ variable "code_bucket" { description = "Bucket where deployable code artifacts are uploaded" } -variable "aws_lambda_layers_account_id" { - type = string - description = "AWS account ID that publishes the ADOT Lambda layers" -} - variable "otel_sample_rate" { type = number description = "OpenTelemetry trace sampling rate — 0.0 = no traces, 1.0 = 100% of traces sampled" diff --git a/infra/modules/aws/lambda_worker/main.tf b/infra/modules/aws/lambda_worker/main.tf index 1061d3e..7a42357 100644 --- a/infra/modules/aws/lambda_worker/main.tf +++ b/infra/modules/aws/lambda_worker/main.tf @@ -1,11 +1,10 @@ module "lambda_worker" { source = "../_shared/lambda" - project_name = var.project_name - environment = var.environment - code_bucket = var.code_bucket - aws_lambda_layers_account_id = var.aws_lambda_layers_account_id - otel_sample_rate = var.otel_sample_rate + project_name = var.project_name + environment = var.environment + code_bucket = var.code_bucket + otel_sample_rate = var.otel_sample_rate lambda_name = local.lambda_name diff --git a/infra/modules/aws/lambda_worker/variables.tf b/infra/modules/aws/lambda_worker/variables.tf index c261cb7..957a700 100644 --- a/infra/modules/aws/lambda_worker/variables.tf +++ b/infra/modules/aws/lambda_worker/variables.tf @@ -14,11 +14,6 @@ variable "code_bucket" { description = "Bucket where deployable code artifacts are uploaded" } -variable "aws_lambda_layers_account_id" { - type = string - description = "AWS account ID that publishes the ADOT Lambda layers" -} - variable "otel_sample_rate" { type = number description = "OpenTelemetry trace sampling rate — 0.0 = no traces, 1.0 = 100% of traces sampled" diff --git a/infra/root.hcl b/infra/root.hcl index 24b3515..99f15a2 100644 --- a/infra/root.hcl +++ b/infra/root.hcl @@ -16,8 +16,6 @@ locals { project_name = element(split("/", local.github_repo), 1) - aws_lambda_layers_account_id = "901920570463" - aws_region = local.global_vars.inputs.aws_region base_reference = "${local.aws_account_id}-${local.aws_region}-${local.project_name}" deploy_role_name = "${local.project_name}-${local.environment}-github-oidc-role" @@ -90,16 +88,15 @@ inputs = merge( local.global_vars.inputs, local.environment_vars.inputs, { - aws_account_id = local.aws_account_id - aws_lambda_layers_account_id = local.aws_lambda_layers_account_id - aws_region = local.aws_region - project_name = local.project_name - environment = local.environment - github_repo = local.github_repo - deploy_role_name = local.deploy_role_name - deploy_role_arn = local.deploy_role_arn - state_bucket = local.state_bucket - state_lock_table = local.state_lock_table - code_bucket = local.code_bucket + aws_account_id = local.aws_account_id + aws_region = local.aws_region + project_name = local.project_name + environment = local.environment + github_repo = local.github_repo + deploy_role_name = local.deploy_role_name + deploy_role_arn = local.deploy_role_arn + state_bucket = local.state_bucket + state_lock_table = local.state_lock_table + code_bucket = local.code_bucket } )