diff --git a/.gitignore b/.gitignore index d73dbdd..adcee4d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ data/ .pgdata +terraform/veda-wfs3-shared-mcp diff --git a/terraform/features-api/ecs_api.tf b/terraform/features-api/ecs_api.tf index 66c9e3b..3c0d835 100644 --- a/terraform/features-api/ecs_api.tf +++ b/terraform/features-api/ecs_api.tf @@ -79,6 +79,10 @@ module "ecs_cluster" { // stupid hack b/c of FastAPI and Starlette bug name = "FAST_API_SCHEME" value = "http" //quick hack for now, TODO: include 'contains' function + }, + { + name = "TIPG_CATALOG_TTL" + value = "300" } ] @@ -97,7 +101,7 @@ module "ecs_cluster" { lb_security_group_id = aws_security_group.web_inbound_sg.id lb_container_port = var.service_port - tags = var.tags + tags = var.tags permissions_boundary_policy_name = var.permissions_boundary_policy_name } diff --git a/terraform/features-api/rds.tf b/terraform/features-api/rds.tf index a2eca28..790432e 100644 --- a/terraform/features-api/rds.tf +++ b/terraform/features-api/rds.tf @@ -45,7 +45,7 @@ resource "aws_db_instance" "db" { db_name = "ghgc" identifier = "${var.project_name}-${var.env}" engine = "postgres" - engine_version = "14.3" + engine_version = "14.9" // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html allocated_storage = 100 max_allocated_storage = 500 diff --git a/terraform/features-api/terraform.tf b/terraform/features-api/terraform.tf index b57e150..bf34d11 100644 --- a/terraform/features-api/terraform.tf +++ b/terraform/features-api/terraform.tf @@ -18,7 +18,7 @@ terraform { } backend "s3" { bucket = "ghgc-smce-tf-shared-state" - key = "root/features-api" + key = "root/features-api-dev" region = "us-west-2" } } diff --git a/terraform/features-api/terraform.tfvars b/terraform/features-api/terraform.tfvars index 18f1643..f4f36ef 100644 --- a/terraform/features-api/terraform.tfvars +++ b/terraform/features-api/terraform.tfvars @@ -1,6 +1,6 @@ -region = "us-west-2" -registry_name = "feature-api-pre-dev" -env = "pre-dev" -project_name = "veda-pre-dev" -service_port = 8080 -vpc_id = "vpc-0c6727f22063d860f" +region = "us-west-2" +registry_name = "feature-api-dev" +env = "smce-ghgc" +project_name = "veda-ghgc-wfs3" +service_port = 8080 +vpc_id = "vpc-0c6727f22063d860f" diff --git a/terraform/modules/aws_ecs_service/main.tf b/terraform/modules/aws_ecs_service/main.tf index aa5684d..651c07a 100755 --- a/terraform/modules/aws_ecs_service/main.tf +++ b/terraform/modules/aws_ecs_service/main.tf @@ -25,9 +25,9 @@ data "aws_iam_policy_document" "ecs_assume_role_policy" { } resource "aws_iam_role" "ecs_execution_role" { - name = "${var.service_name}-${var.environment}_ecs_task_execution_role" - assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json - tags = var.tags + name = "${var.service_name}-${var.environment}_ecs_task_execution_role" + assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json + tags = var.tags permissions_boundary = local.permissions_boundary } @@ -123,15 +123,15 @@ resource "aws_security_group_rule" "service_egress" { // bind the ECS service's SG as a source // to the VPC's default SG if it was passed as a variable resource "aws_security_group_rule" "rds_sg_allows_ecs_sg" { - for_each = { - for index, rule in var.additional_sg_ingress_rules_for_vpc_default_sg: + for_each = { + for index, rule in var.additional_sg_ingress_rules_for_vpc_default_sg : rule.primary_key => rule # this works b/c one key has to be primary } - security_group_id = each.value.vpc_default_sg_id - type = "ingress" - from_port = each.value.from_port - to_port = each.value.to_port - protocol = each.value.protocol + security_group_id = each.value.vpc_default_sg_id + type = "ingress" + from_port = each.value.from_port + to_port = each.value.to_port + protocol = each.value.protocol source_security_group_id = aws_security_group.service.id } @@ -173,7 +173,7 @@ resource "aws_security_group_rule" "service_ingress_lb" { # ECS ######################################################################## resource "aws_ecs_cluster" "service" { - name = "tf-${var.service_name}-${var.environment}" + name = "tf-${var.service_name}-${var.environment}-dev" tags = var.tags setting { name = "containerInsights" @@ -196,8 +196,8 @@ resource "aws_ecs_service" "service" { deployment_minimum_healthy_percent = 100 network_configuration { - subnets = var.subnet_ids - security_groups = [aws_security_group.service.id] + subnets = var.subnet_ids + security_groups = [aws_security_group.service.id] //assign_public_ip = true } @@ -220,22 +220,22 @@ resource "aws_ecs_task_definition" "service" { tags = var.tags execution_role_arn = aws_iam_role.ecs_execution_role.arn task_role_arn = aws_iam_role.ecs_execution_role.arn - container_definitions = templatefile("${path.module}/container_definition.tftpl", - { - service_name = var.service_name - environment = var.environment - image = var.image - container_command = length(var.container_command) > 0 ? jsonencode(var.container_command) : "" - working_directory = var.container_working_directory - container_secrets = jsonencode(var.container_secrets) - container_environment = jsonencode(var.container_environment) - service_protocol = var.service_protocol - service_port = var.service_port - use_adot_as_sidecar = var.use_adot_as_sidecar ? "on" : "" - log_group = aws_cloudwatch_log_group.service.name - region = var.region - } - ) + container_definitions = templatefile("${path.module}/container_definition.tftpl", + { + service_name = var.service_name + environment = var.environment + image = var.image + container_command = length(var.container_command) > 0 ? jsonencode(var.container_command) : "" + working_directory = var.container_working_directory + container_secrets = jsonencode(var.container_secrets) + container_environment = jsonencode(var.container_environment) + service_protocol = var.service_protocol + service_port = var.service_port + use_adot_as_sidecar = var.use_adot_as_sidecar ? "on" : "" + log_group = aws_cloudwatch_log_group.service.name + region = var.region + } + ) } ####################################################################### @@ -247,19 +247,19 @@ resource "aws_ecs_task_definition" "service" { data "aws_iam_policy_document" "api_ecs_to_otel_access" { statement { actions = [ - "xray:PutTraceSegments", - "xray:PutTelemetryRecords", - "xray:GetSamplingRules", - "xray:GetSamplingTargets", - "xray:GetSamplingStatisticSummaries", - "cloudwatch:PutMetricData", - "ec2:DescribeVolumes", - "ec2:DescribeTags", - "ssm:GetParameters" + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + "xray:GetSamplingRules", + "xray:GetSamplingTargets", + "xray:GetSamplingStatisticSummaries", + "cloudwatch:PutMetricData", + "ec2:DescribeVolumes", + "ec2:DescribeTags", + "ssm:GetParameters" ] resources = [ - "*", + "*", ] } } diff --git a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py b/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py new file mode 100644 index 0000000..806e49d --- /dev/null +++ b/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py @@ -0,0 +1,72 @@ +import boto3 +import http.client +import os +import base64 +import ast +import json +mwaa_env_name = 'veda-pipeline-staging-mwaa' +dag_name = 'veda_discover' +mwaa_cli_command = 'dags trigger' +client = boto3.client('mwaa') + + +def lambda_handler(event, context): + for record in event['Records']: + print(f"[ RECORD ]: {record}") + s3_event_key = record['s3']['object']['key'] + print(f"[ S3 EVENT KEY ]: {s3_event_key}") + s3_filename_target = os.path.split(s3_event_key)[-1] + print(f"[ S3 FILENAME TARGET ]: {s3_filename_target}") + s3_filename_no_ext = os.path.splitext(s3_filename_target)[0] + print(f"[ S3 FILENAME NO EXT ]: {s3_filename_no_ext}") + + if s3_filename_target.endswith(".gpkg"): + return { + 'statusCode': 200, + 'body': json.dumps('Hello from Lambda!') + } + + if s3_event_key.startswith("EIS/FEDSoutput-v3"): + return { + 'statusCode': 200, + 'body': json.dumps('Hello from Lambda!') + } + + bucket_key_prefix = "EIS/FEDSoutput/Snapshot/" + if s3_event_key.startswith("EIS/FEDSoutput-v3"): + bucket_key_prefix = "EIS/FEDSoutput-v3/Snapshot/" + if s3_filename_no_ext.startswith("lf_"): + bucket_key_prefix = "EIS/FEDSoutput/LFArchive/" + + + # get web token + mwaa_cli_token = client.create_cli_token( + Name=mwaa_env_name + ) + print(f"[ CLI TOKEN ]: {mwaa_cli_token}") + serialized_args = json.dumps({ + "discovery": "s3", + "collection": s3_filename_no_ext, + "prefix": bucket_key_prefix, + "bucket": "veda-data-store-staging", + "filename_regex": f"^(.*){s3_filename_target}$", + "vector": True + }) + conn = http.client.HTTPSConnection(mwaa_cli_token['WebServerHostname']) + payload = f"{mwaa_cli_command} {dag_name} --conf '{serialized_args}'" + print(f"[ CLI PAYLOAD ]: {payload}") + headers = { + 'Authorization': 'Bearer ' + mwaa_cli_token['CliToken'], + 'Content-Type': 'text/plain' + } + conn.request("POST", "/aws_mwaa/cli", payload, headers) + res = conn.getresponse() + data = res.read() + dict_str = data.decode("UTF-8") + mydata = ast.literal_eval(dict_str) + print(f"[ DATA ]: {mydata}") + print(f"[ STDOUT ]: {base64.b64decode(mydata['stdout'])}") + return { + 'statusCode': 200, + 'body': json.dumps('Hello from Lambda!') + } diff --git a/terraform/veda-wfs3/rds.tf b/terraform/veda-wfs3/rds.tf new file mode 100644 index 0000000..05db52c --- /dev/null +++ b/terraform/veda-wfs3/rds.tf @@ -0,0 +1,66 @@ +resource "aws_db_subnet_group" "db" { + name = "tf-${var.project_name}-${var.env}-subnet-group" + subnet_ids = module.networking.private_subnets_id + tags = { + Name = "tf-${var.project_name}-subnet-group" + } +} + +resource "aws_db_parameter_group" "default" { + name = "tf-${var.project_name}-${var.env}-postgres14-param-group" + family = "postgres14" + + parameter { + name = "work_mem" + # NOTE: I had `work_mem` set to ~100MB and `max_connections` around 75 and TileJSON completely failed + # 16MB + value = var.env == "staging" ? "16384" : "8192" + } + + parameter { + name = "max_connections" + value = "475" + apply_method = "pending-reboot" + } + +# NOTE: here to show what shared_buffers are but doesn't really make sense why it won't provision with these +# parameter { +# name = "shared_buffers" +# value = var.env == "staging" ? "8064856" : "4032428" +# apply_method = "pending-reboot" +# } + + parameter { + name = "seq_page_cost" + value = "1" + } + + parameter { + name = "random_page_cost" + value = "1.2" + } +} + +resource "aws_db_instance" "db" { + db_name = "veda" + identifier = "${var.project_name}-${var.env}" + engine = "postgres" + engine_version = "14.10" + // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html + allocated_storage = 100 + max_allocated_storage = 500 + storage_type = "gp2" + instance_class = var.env == "staging" ? "db.r5.xlarge" : "db.r5.large" + db_subnet_group_name = aws_db_subnet_group.db.name + vpc_security_group_ids = module.networking.security_groups_ids + skip_final_snapshot = true + apply_immediately = true + backup_retention_period = 7 + username = "postgres" + password = var.db_password + storage_encrypted = var.db_encrypted + allow_major_version_upgrade = true + parameter_group_name = aws_db_parameter_group.default.name +} + + diff --git a/terraform/veda-wfs3/s3_event_bridge_lambda.tf b/terraform/veda-wfs3/s3_event_bridge_lambda.tf new file mode 100644 index 0000000..b618abf --- /dev/null +++ b/terraform/veda-wfs3/s3_event_bridge_lambda.tf @@ -0,0 +1,173 @@ +##################################################### +# Execution Role +##################################################### +resource "aws_iam_role" "lambda_exec_role" { + provider = aws.west2 + name = "lambda-exec-role-s3-event-bridge-${var.project_name}-${var.env}" + tags = var.tags + + assume_role_policy = <