From e797c48059ed7a6d4b181ad3d84ada2725d7d3d9 Mon Sep 17 00:00:00 2001 From: sudobangbang Date: Fri, 9 Jun 2023 11:06:57 -0700 Subject: [PATCH 1/6] apply ttl (#24) Co-authored-by: ranchodeluxe --- terraform/veda-wfs3/ecs_api.tf | 4 ++++ veda-wfs3-app/fast_api_main.py | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/terraform/veda-wfs3/ecs_api.tf b/terraform/veda-wfs3/ecs_api.tf index a791f6e..1edbdfe 100644 --- a/terraform/veda-wfs3/ecs_api.tf +++ b/terraform/veda-wfs3/ecs_api.tf @@ -68,6 +68,10 @@ module "ecs_cluster" { // stupid hack b/c of FastAPI and Starlette bug name = "FAST_API_SCHEME" value = var.env == "west2-staging" ? "https" : "http" + }, + { + name = "TIPG_CATALOG_TTL" + value = "300" } ] diff --git a/veda-wfs3-app/fast_api_main.py b/veda-wfs3-app/fast_api_main.py index 4fb55b3..3adada4 100644 --- a/veda-wfs3-app/fast_api_main.py +++ b/veda-wfs3-app/fast_api_main.py @@ -13,7 +13,8 @@ from starlette.middleware.cors import CORSMiddleware from tipg.db import close_db_connection, connect_to_db, register_collection_catalog from tipg.factory import Endpoints as FeaturesEndpoints -from tipg.settings import PostgresSettings +from tipg.settings import PostgresSettings, APISettings, DatabaseSettings +from tipg.middleware import CatalogUpdateMiddleware from typing import Callable logger = logging.getLogger(__name__) @@ -91,6 +92,8 @@ async def dispatch(self, request, call_next): "postgres_port": db_config["port"], "postgres_dbname": db_config["dbname"] }) +api_settings = APISettings() +db_settings = DatabaseSettings() endpoints = FeaturesEndpoints(router=APIRouter(route_class=LoggerRouteHandler)) app.include_router(endpoints.router, tags=["OGC Features"]) @@ -103,6 +106,19 @@ async def dispatch(self, request, call_next): allow_methods=["GET", "POST", "OPTIONS"], allow_headers=["*"], ) +if api_settings.catalog_ttl: + app.add_middleware( + CatalogUpdateMiddleware, + ttl=api_settings.catalog_ttl, + schemas=db_settings.schemas, + tables=db_settings.tables, + exclude_tables=db_settings.exclude_tables, + exclude_table_schemas=db_settings.exclude_table_schemas, + functions=db_settings.functions, + exclude_functions=db_settings.exclude_functions, + exclude_function_schemas=db_settings.exclude_function_schemas, + spatial=db_settings.only_spatial_tables, + ) @app.on_event("startup") From a238b8ba5d88f18575c78087e8386c82a74b1d30 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Mon, 21 Aug 2023 08:12:06 -0700 Subject: [PATCH 2/6] include RDS upgrade version --- terraform/veda-wfs3/rds.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/veda-wfs3/rds.tf b/terraform/veda-wfs3/rds.tf index e75f047..d54bf83 100644 --- a/terraform/veda-wfs3/rds.tf +++ b/terraform/veda-wfs3/rds.tf @@ -45,7 +45,7 @@ resource "aws_db_instance" "db" { db_name = "veda" identifier = "${var.project_name}-${var.env}" engine = "postgres" - engine_version = "14.3" + engine_version = "14.7" // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html allocated_storage = 100 max_allocated_storage = 500 From 65469a92f63c21dd682f334bc517a0ae156d65cc Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Mon, 29 Jan 2024 12:06:18 -0600 Subject: [PATCH 3/6] Support encrypted RDS instance (#28) --- terraform/veda-wfs3/rds.tf | 1 + terraform/veda-wfs3/variables.tf | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/terraform/veda-wfs3/rds.tf b/terraform/veda-wfs3/rds.tf index d54bf83..e6b5cc5 100644 --- a/terraform/veda-wfs3/rds.tf +++ b/terraform/veda-wfs3/rds.tf @@ -58,6 +58,7 @@ resource "aws_db_instance" "db" { backup_retention_period = 7 username = "postgres" password = var.db_password + storage_encrypted = var.db_encrypted allow_major_version_upgrade = true parameter_group_name = aws_db_parameter_group.default.name } diff --git a/terraform/veda-wfs3/variables.tf b/terraform/veda-wfs3/variables.tf index 7645237..9d459e9 100755 --- a/terraform/veda-wfs3/variables.tf +++ b/terraform/veda-wfs3/variables.tf @@ -37,6 +37,12 @@ variable "db_password" { sensitive = true } +variable "db_encrypted" { + description = "Whether RDS storage should be encrypted" + type = bool + default = false +} + variable "dns_zone_name" { } From 857a3e01a55d3eab822a086b4531ea7723f8793a Mon Sep 17 00:00:00 2001 From: sudobangbang Date: Wed, 22 May 2024 16:48:34 -0600 Subject: [PATCH 4/6] RDS and S3 Changes (#29) * .gitignore * rds and s3 changes * changes --------- Co-authored-by: ranchodeluxe --- .gitignore | 1 + .../lambda_function.py | 17 ++++++++++++++++- terraform/veda-wfs3/rds.tf | 2 +- terraform/veda-wfs3/s3_event_bridge_lambda.tf | 8 +++++++- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d73dbdd..adcee4d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ data/ .pgdata +terraform/veda-wfs3-shared-mcp diff --git a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py b/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py index d524d83..806e49d 100644 --- a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py +++ b/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py @@ -20,10 +20,25 @@ def lambda_handler(event, context): s3_filename_no_ext = os.path.splitext(s3_filename_target)[0] print(f"[ S3 FILENAME NO EXT ]: {s3_filename_no_ext}") + if s3_filename_target.endswith(".gpkg"): + return { + 'statusCode': 200, + 'body': json.dumps('Hello from Lambda!') + } + + if s3_event_key.startswith("EIS/FEDSoutput-v3"): + return { + 'statusCode': 200, + 'body': json.dumps('Hello from Lambda!') + } + bucket_key_prefix = "EIS/FEDSoutput/Snapshot/" + if s3_event_key.startswith("EIS/FEDSoutput-v3"): + bucket_key_prefix = "EIS/FEDSoutput-v3/Snapshot/" if s3_filename_no_ext.startswith("lf_"): bucket_key_prefix = "EIS/FEDSoutput/LFArchive/" + # get web token mwaa_cli_token = client.create_cli_token( Name=mwaa_env_name @@ -54,4 +69,4 @@ def lambda_handler(event, context): return { 'statusCode': 200, 'body': json.dumps('Hello from Lambda!') - } \ No newline at end of file + } diff --git a/terraform/veda-wfs3/rds.tf b/terraform/veda-wfs3/rds.tf index e6b5cc5..05db52c 100644 --- a/terraform/veda-wfs3/rds.tf +++ b/terraform/veda-wfs3/rds.tf @@ -45,7 +45,7 @@ resource "aws_db_instance" "db" { db_name = "veda" identifier = "${var.project_name}-${var.env}" engine = "postgres" - engine_version = "14.7" + engine_version = "14.10" // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html allocated_storage = 100 max_allocated_storage = 500 diff --git a/terraform/veda-wfs3/s3_event_bridge_lambda.tf b/terraform/veda-wfs3/s3_event_bridge_lambda.tf index 77ae894..b618abf 100644 --- a/terraform/veda-wfs3/s3_event_bridge_lambda.tf +++ b/terraform/veda-wfs3/s3_event_bridge_lambda.tf @@ -141,9 +141,15 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda_exec_role.arn handler = "lambda_function.lambda_handler" source_code_hash = data.archive_file.archive.output_base64sha256 - runtime = "python3.7" + runtime = "python3.9" publish = true tags = var.tags + + environment { + variables = { + LOG_GROUP_NAME = "/aws/lambda/s3-event-bridge-to-sfn-execute-${var.project_name}-${var.env}" + } + } } resource "aws_cloudwatch_log_group" "group" { From c6a03ff914db96a3fefe4e75dd709275e5cf241f Mon Sep 17 00:00:00 2001 From: anisbhsl Date: Tue, 16 Jul 2024 16:06:44 -0500 Subject: [PATCH 5/6] update terraform --- terraform/features-api/rds.tf | 2 +- terraform/features-api/terraform.tf | 2 +- terraform/features-api/terraform.tfvars | 12 ++-- terraform/modules/aws_ecs_service/main.tf | 78 +++++++++++------------ 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/terraform/features-api/rds.tf b/terraform/features-api/rds.tf index a2eca28..790432e 100644 --- a/terraform/features-api/rds.tf +++ b/terraform/features-api/rds.tf @@ -45,7 +45,7 @@ resource "aws_db_instance" "db" { db_name = "ghgc" identifier = "${var.project_name}-${var.env}" engine = "postgres" - engine_version = "14.3" + engine_version = "14.9" // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html allocated_storage = 100 max_allocated_storage = 500 diff --git a/terraform/features-api/terraform.tf b/terraform/features-api/terraform.tf index b57e150..bf34d11 100644 --- a/terraform/features-api/terraform.tf +++ b/terraform/features-api/terraform.tf @@ -18,7 +18,7 @@ terraform { } backend "s3" { bucket = "ghgc-smce-tf-shared-state" - key = "root/features-api" + key = "root/features-api-dev" region = "us-west-2" } } diff --git a/terraform/features-api/terraform.tfvars b/terraform/features-api/terraform.tfvars index 18f1643..f4f36ef 100644 --- a/terraform/features-api/terraform.tfvars +++ b/terraform/features-api/terraform.tfvars @@ -1,6 +1,6 @@ -region = "us-west-2" -registry_name = "feature-api-pre-dev" -env = "pre-dev" -project_name = "veda-pre-dev" -service_port = 8080 -vpc_id = "vpc-0c6727f22063d860f" +region = "us-west-2" +registry_name = "feature-api-dev" +env = "smce-ghgc" +project_name = "veda-ghgc-wfs3" +service_port = 8080 +vpc_id = "vpc-0c6727f22063d860f" diff --git a/terraform/modules/aws_ecs_service/main.tf b/terraform/modules/aws_ecs_service/main.tf index aa5684d..651c07a 100755 --- a/terraform/modules/aws_ecs_service/main.tf +++ b/terraform/modules/aws_ecs_service/main.tf @@ -25,9 +25,9 @@ data "aws_iam_policy_document" "ecs_assume_role_policy" { } resource "aws_iam_role" "ecs_execution_role" { - name = "${var.service_name}-${var.environment}_ecs_task_execution_role" - assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json - tags = var.tags + name = "${var.service_name}-${var.environment}_ecs_task_execution_role" + assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json + tags = var.tags permissions_boundary = local.permissions_boundary } @@ -123,15 +123,15 @@ resource "aws_security_group_rule" "service_egress" { // bind the ECS service's SG as a source // to the VPC's default SG if it was passed as a variable resource "aws_security_group_rule" "rds_sg_allows_ecs_sg" { - for_each = { - for index, rule in var.additional_sg_ingress_rules_for_vpc_default_sg: + for_each = { + for index, rule in var.additional_sg_ingress_rules_for_vpc_default_sg : rule.primary_key => rule # this works b/c one key has to be primary } - security_group_id = each.value.vpc_default_sg_id - type = "ingress" - from_port = each.value.from_port - to_port = each.value.to_port - protocol = each.value.protocol + security_group_id = each.value.vpc_default_sg_id + type = "ingress" + from_port = each.value.from_port + to_port = each.value.to_port + protocol = each.value.protocol source_security_group_id = aws_security_group.service.id } @@ -173,7 +173,7 @@ resource "aws_security_group_rule" "service_ingress_lb" { # ECS ######################################################################## resource "aws_ecs_cluster" "service" { - name = "tf-${var.service_name}-${var.environment}" + name = "tf-${var.service_name}-${var.environment}-dev" tags = var.tags setting { name = "containerInsights" @@ -196,8 +196,8 @@ resource "aws_ecs_service" "service" { deployment_minimum_healthy_percent = 100 network_configuration { - subnets = var.subnet_ids - security_groups = [aws_security_group.service.id] + subnets = var.subnet_ids + security_groups = [aws_security_group.service.id] //assign_public_ip = true } @@ -220,22 +220,22 @@ resource "aws_ecs_task_definition" "service" { tags = var.tags execution_role_arn = aws_iam_role.ecs_execution_role.arn task_role_arn = aws_iam_role.ecs_execution_role.arn - container_definitions = templatefile("${path.module}/container_definition.tftpl", - { - service_name = var.service_name - environment = var.environment - image = var.image - container_command = length(var.container_command) > 0 ? jsonencode(var.container_command) : "" - working_directory = var.container_working_directory - container_secrets = jsonencode(var.container_secrets) - container_environment = jsonencode(var.container_environment) - service_protocol = var.service_protocol - service_port = var.service_port - use_adot_as_sidecar = var.use_adot_as_sidecar ? "on" : "" - log_group = aws_cloudwatch_log_group.service.name - region = var.region - } - ) + container_definitions = templatefile("${path.module}/container_definition.tftpl", + { + service_name = var.service_name + environment = var.environment + image = var.image + container_command = length(var.container_command) > 0 ? jsonencode(var.container_command) : "" + working_directory = var.container_working_directory + container_secrets = jsonencode(var.container_secrets) + container_environment = jsonencode(var.container_environment) + service_protocol = var.service_protocol + service_port = var.service_port + use_adot_as_sidecar = var.use_adot_as_sidecar ? "on" : "" + log_group = aws_cloudwatch_log_group.service.name + region = var.region + } + ) } ####################################################################### @@ -247,19 +247,19 @@ resource "aws_ecs_task_definition" "service" { data "aws_iam_policy_document" "api_ecs_to_otel_access" { statement { actions = [ - "xray:PutTraceSegments", - "xray:PutTelemetryRecords", - "xray:GetSamplingRules", - "xray:GetSamplingTargets", - "xray:GetSamplingStatisticSummaries", - "cloudwatch:PutMetricData", - "ec2:DescribeVolumes", - "ec2:DescribeTags", - "ssm:GetParameters" + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + "xray:GetSamplingRules", + "xray:GetSamplingTargets", + "xray:GetSamplingStatisticSummaries", + "cloudwatch:PutMetricData", + "ec2:DescribeVolumes", + "ec2:DescribeTags", + "ssm:GetParameters" ] resources = [ - "*", + "*", ] } } From 69ac5c9b5a1c719222dab3d82b6fb7cc2dd64277 Mon Sep 17 00:00:00 2001 From: anisbhsl Date: Tue, 16 Jul 2024 16:21:17 -0500 Subject: [PATCH 6/6] added refresh ended back --- wfs3-app/fast_api_main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/wfs3-app/fast_api_main.py b/wfs3-app/fast_api_main.py index dcf98ff..3adada4 100644 --- a/wfs3-app/fast_api_main.py +++ b/wfs3-app/fast_api_main.py @@ -146,13 +146,13 @@ async def ping(): return JSONResponse(status_code=200, content={"ping": "pong"}) -# @app.get("/refresh") -# async def refresh(request: Request): -# """Return parsed catalog data for testing.""" -# with tracer.start_as_current_span("refresh"): -# refresh_counter.add(1, {"refresh": "count"}) -# await connect_to_db(app, settings=postgresql_settings) -# await register_collection_catalog(app) -# return JSONResponse(status_code=200, content={"status": "refreshed"}) +@app.get("/refresh") +async def refresh(request: Request): + """Return parsed catalog data for testing.""" + with tracer.start_as_current_span("refresh"): + refresh_counter.add(1, {"refresh": "count"}) + await connect_to_db(app, settings=postgresql_settings) + await register_collection_catalog(app) + return JSONResponse(status_code=200, content={"status": "refreshed"}) FastAPIInstrumentor.instrument_app(app, excluded_urls="/conformance,/healthz")