Skip to content

ENG-2635: Standardize RDS & MySQL namespace meta field names #7589

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JadeCara merged 8 commits into from
Mar 11, 2026
Merged

ENG-2635: Standardize RDS & MySQL namespace meta field names #7589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5b58d93
Standardize RDS & MySQL namespace meta field names
Mar 6, 2026
2e8754e
Add changelog entry for #7589
Mar 6, 2026
267226f
Rename changelog file with descriptive slug
Mar 6, 2026
4a370da
Merge branch 'main' into ENG-2635-rds-namespace-field-migration
JadeCara Mar 9, 2026
00f10b9
Address PR review feedback
Mar 9, 2026
61ec714
Merge branch 'main' into ENG-2635-rds-namespace-field-migration
JadeCara Mar 9, 2026
75a0f2c
Merge branch 'main' into ENG-2635-rds-namespace-field-migration
JadeCara Mar 10, 2026
7baa155
fix down migration
Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions changelog/7589-rds-namespace-field-rename.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
type: Changed
description: Standardize RDS and MySQL namespace meta field names (database_id → database_name, database_instance_id → database_instance_name) in connector and data migration
pr: 7589
labels: ["db-migration"]
167 changes: 167 additions & 0 deletions ...c/migrations/versions/xx_2026_03_06_2033_04281f44cc0b_rename_rds_namespace_meta_fields.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
"""rename rds namespace meta fields
Copy link
Copy Markdown
Contributor

@adamsachs adamsachs Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i'd just provide the explicit before/after 'mapping' of field names explicitly here in the docstring for extra visibility

Copy link
Copy Markdown
Contributor Author

@JadeCara JadeCara Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done — updated the docstring with an explicit before/after field mapping table.


Renames the following fields in ctl_datasets.fides_meta namespace metadata
for rds_postgres, rds_mysql, and mysql connection types:

database_id → database_name
database_instance_id → database_instance_name

Revision ID: 04281f44cc0b
Revises: e3a9f1b2c4d5
Create Date: 2026-03-06 20:33:13.839629

"""

from alembic import op

# revision identifiers, used by Alembic.
revision = "04281f44cc0b"
down_revision = "e3a9f1b2c4d5"
branch_labels = None
depends_on = None


def upgrade():
# Rename database_id → database_name in RDS Postgres namespace metadata.
# The namespace dict is stored as JSON inside fides_meta on ctl_datasets.
# We target only rows with connection_type = 'rds_postgres'.
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_id}',
'{namespace, database_name}',
fides_meta::jsonb #> '{namespace, database_id}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'rds_postgres'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_id';
"""
)

# Rename database_instance_id → database_instance_name.
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_instance_id}',
'{namespace, database_instance_name}',
fides_meta::jsonb #> '{namespace, database_instance_id}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'rds_postgres'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_instance_id';
"""
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.

# Rename database_id → database_name in RDS MySQL namespace metadata.
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_id}',
'{namespace, database_name}',
fides_meta::jsonb #> '{namespace, database_id}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'rds_mysql'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_id';
"""
)

# Rename database_instance_id → database_instance_name in RDS MySQL.
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_instance_id}',
'{namespace, database_instance_name}',
fides_meta::jsonb #> '{namespace, database_instance_id}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'rds_mysql'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_instance_id';
"""
)

# Rename database_id → database_name in plain MySQL namespace metadata.
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_id}',
'{namespace, database_name}',
fides_meta::jsonb #> '{namespace, database_id}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'mysql'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_id';
"""
)


def downgrade():
# Reverse: database_name → database_id
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_name}',
'{namespace, database_id}',
fides_meta::jsonb #> '{namespace, database_name}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'rds_postgres'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_name';
"""
)

# Reverse: database_instance_name → database_instance_id
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_instance_name}',
'{namespace, database_instance_id}',
fides_meta::jsonb #> '{namespace, database_instance_name}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'rds_postgres'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_instance_name';
"""
)

# Reverse RDS MySQL: database_name → database_id
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_name}',
'{namespace, database_id}',
fides_meta::jsonb #> '{namespace, database_name}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'rds_mysql'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_name';
"""
)

# Reverse RDS MySQL: database_instance_name → database_instance_id
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_instance_name}',
'{namespace, database_instance_id}',
fides_meta::jsonb #> '{namespace, database_instance_name}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'rds_mysql'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_instance_name';
"""
)

# Reverse plain MySQL: database_name → database_id
op.execute(
"""
UPDATE ctl_datasets
SET fides_meta = jsonb_set(
fides_meta::jsonb #- '{namespace, database_name}',
'{namespace, database_id}',
fides_meta::jsonb #> '{namespace, database_name}'
)
WHERE (fides_meta::jsonb #>> '{namespace, connection_type}') = 'mysql'
AND (fides_meta::jsonb #> '{namespace}') ? 'database_name';
"""
)
4 changes: 2 additions & 2 deletions src/fides/api/service/connectors/rds_mysql_connector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ def create_client(self) -> Engine:
"Namespace meta is not set. Please call pre_client_creation_hook before creating the client."
)

database_instance_name = self.namespace_meta["database_instance_id"]
db_name = self.namespace_meta["database_id"]
database_instance_name = self.namespace_meta["database_instance_name"]
db_name = self.namespace_meta["database_name"]
db_info = self.get_database_instance_connection_info(database_instance_name)
return self.create_engine(
db_username=self.typed_secrets.db_username,
Expand Down
4 changes: 2 additions & 2 deletions src/fides/api/service/connectors/rds_postgres_connector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ def create_client(self) -> Engine:
"Namespace meta is not set. Please call pre_client_creation_hook before creating the client."
)

database_instance_name = self.namespace_meta["database_instance_id"]
db_name = self.namespace_meta["database_id"]
database_instance_name = self.namespace_meta["database_instance_name"]
db_name = self.namespace_meta["database_name"]
db_info = self.get_database_instance_connection_info(database_instance_name)
return self.create_engine(
db_username=self.typed_secrets.db_username,
Expand Down
Loading