diff --git a/.github/workflows/merge-upstream.yml b/.github/workflows/merge-upstream.yml
index e07c4f0a27..e2af037b56 100644
--- a/.github/workflows/merge-upstream.yml
+++ b/.github/workflows/merge-upstream.yml
@@ -45,6 +45,9 @@ jobs:
           git checkout HEAD -- .github/
           git reset HEAD .github/
 
+          # Keep "deleted by us" files deleted
+          git status --porcelain | grep '^DU' | cut -c 4- | xargs -r git rm
+
           # Commit the merge with conflict markers
           git commit -a -m "Merge upstream changes from ${{ github.event.inputs.upstream_ref }}"