Nightly E2E Resource Cleanup #19
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly E2E Resource Cleanup | |
| on: | |
| schedule: | |
| - cron: '0 2 * * *' # 02:00 UTC every day | |
| workflow_dispatch: # allow manual trigger | |
| jobs: | |
| cleanup: | |
| name: Sweep orphaned E2E test resources | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - run: pip install boto3 | |
| # ── Single account ──────────────────────────────────────────────────── | |
| - name: Assume role — single account | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_SINGLE_ACCOUNT_ID }}:role/GitHubActionsE2ERole | |
| aws-region: ap-northeast-2 | |
| continue-on-error: true | |
| # Guard: if any map-auto-tagger-e2e-pr* stack in this account is younger | |
| # than the in-flight window or actively IN_PROGRESS, skip the sweep to | |
| # avoid deleting a live E2E run's resources mid-test. Stacks skipped | |
| # tonight are caught tomorrow night once they age past the window. | |
| - name: Guard — detect in-flight E2E (single account) | |
| id: guard_single | |
| run: | | |
| if python3 .github/scripts/nightly_cleanup_guard.py check-account; then | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| fi | |
| continue-on-error: true | |
| - name: Delete tagged resources — single account | |
| if: steps.guard_single.outputs.skip != 'true' | |
| working-directory: .github/scripts | |
| run: | | |
| # Sweep resources tagged with either the Lambda-applied | |
| # `map-migrated` key or the pre-applied `e2e-run-id` key (PR #7.a | |
| # switched E2E pre-tagging from map-migrated to e2e-run-id so | |
| # verify_tags actually tests the Lambda). Empty --tag-value means | |
| # "match any value" — covers every past run's per-run MPE/run-id tag. | |
| python3 teardown.py \ | |
| --all \ | |
| --sweep-keys map-migrated,e2e-run-id \ | |
| --regions ap-northeast-2,us-east-1,us-west-2 | |
| continue-on-error: true | |
| - name: Delete stale CF stacks — single account (ap-northeast-2) | |
| if: steps.guard_single.outputs.skip != 'true' | |
| run: | | |
| for stack in $(aws cloudformation list-stacks \ | |
| --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE \ | |
| ROLLBACK_COMPLETE UPDATE_ROLLBACK_COMPLETE \ | |
| CREATE_FAILED UPDATE_FAILED ROLLBACK_FAILED \ | |
| --query 'StackSummaries[?starts_with(StackName, `map-auto-tagger-e2e-pr`)].StackName' \ | |
| --output text \ | |
| --region ap-northeast-2 2>/dev/null); do | |
| echo "Deleting stack: $stack" | |
| aws cloudformation delete-stack --stack-name "$stack" --region ap-northeast-2 || true | |
| done | |
| continue-on-error: true | |
| - name: Delete stale CF stacks — single account (us-east-1) | |
| if: steps.guard_single.outputs.skip != 'true' | |
| run: | | |
| for stack in $(aws cloudformation list-stacks \ | |
| --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE \ | |
| ROLLBACK_COMPLETE UPDATE_ROLLBACK_COMPLETE \ | |
| CREATE_FAILED UPDATE_FAILED ROLLBACK_FAILED \ | |
| --query 'StackSummaries[?starts_with(StackName, `map-auto-tagger-e2e-pr`)].StackName' \ | |
| --output text \ | |
| --region us-east-1 2>/dev/null); do | |
| echo "Deleting stack: $stack" | |
| aws cloudformation delete-stack --stack-name "$stack" --region us-east-1 || true | |
| done | |
| continue-on-error: true | |
| - name: Delete stale CF stacks — single account (us-west-2) | |
| if: steps.guard_single.outputs.skip != 'true' | |
| run: | | |
| for stack in $(aws cloudformation list-stacks \ | |
| --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE \ | |
| ROLLBACK_COMPLETE UPDATE_ROLLBACK_COMPLETE \ | |
| CREATE_FAILED UPDATE_FAILED ROLLBACK_FAILED \ | |
| --query 'StackSummaries[?starts_with(StackName, `map-auto-tagger-e2e-pr`)].StackName' \ | |
| --output text \ | |
| --region us-west-2 2>/dev/null); do | |
| echo "Deleting stack: $stack" | |
| aws cloudformation delete-stack --stack-name "$stack" --region us-west-2 || true | |
| done | |
| continue-on-error: true | |
| # ── Management account (StackSets live here) ────────────────────────── | |
| - name: Assume role — management account | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_MGMT_ACCOUNT_ID }}:role/GitHubActionsE2ERole | |
| aws-region: ap-northeast-2 | |
| continue-on-error: true | |
| - name: Guard — detect in-flight E2E (management account) | |
| id: guard_mgmt | |
| run: | | |
| if python3 .github/scripts/nightly_cleanup_guard.py check-account; then | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| fi | |
| continue-on-error: true | |
| - name: Delete tagged resources — management account | |
| if: steps.guard_mgmt.outputs.skip != 'true' | |
| working-directory: .github/scripts | |
| run: | | |
| # Sweep resources tagged with either the Lambda-applied | |
| # `map-migrated` key or the pre-applied `e2e-run-id` key (PR #7.a | |
| # switched E2E pre-tagging from map-migrated to e2e-run-id so | |
| # verify_tags actually tests the Lambda). Empty --tag-value means | |
| # "match any value" — covers every past run's per-run MPE/run-id tag. | |
| python3 teardown.py \ | |
| --all \ | |
| --sweep-keys map-migrated,e2e-run-id \ | |
| --regions ap-northeast-2,us-east-1,us-west-2 | |
| continue-on-error: true | |
| - name: Delete stale StackSets — management account | |
| if: steps.guard_mgmt.outputs.skip != 'true' | |
| run: | | |
| for ss in $(aws cloudformation list-stack-sets \ | |
| --status ACTIVE \ | |
| --query 'Summaries[?starts_with(StackSetName, `map-auto-tagger-e2e-pr`)].StackSetName' \ | |
| --output text \ | |
| --region ap-northeast-2 2>/dev/null); do | |
| echo "Removing all instances from StackSet: $ss" | |
| # Delete all stack instances first (required before deleting the StackSet) | |
| aws cloudformation delete-stack-instances \ | |
| --stack-set-name "$ss" \ | |
| --regions ap-northeast-2 \ | |
| --no-retain-stacks \ | |
| --deployment-targets 'OrganizationalUnitIds=[]' \ | |
| --region ap-northeast-2 2>/dev/null || \ | |
| aws cloudformation delete-stack-instances \ | |
| --stack-set-name "$ss" \ | |
| --accounts \ | |
| "${{ secrets.AWS_LINKED1_ACCOUNT_ID }}" \ | |
| "${{ secrets.AWS_LINKED2_ACCOUNT_ID }}" \ | |
| "${{ secrets.AWS_LINKED3_ACCOUNT_ID }}" \ | |
| "${{ secrets.AWS_LINKED4_ACCOUNT_ID }}" \ | |
| "${{ secrets.AWS_LINKED5_ACCOUNT_ID }}" \ | |
| --regions ap-northeast-2 \ | |
| --no-retain-stacks \ | |
| --region ap-northeast-2 2>/dev/null || true | |
| sleep 5 | |
| echo "Deleting StackSet: $ss" | |
| aws cloudformation delete-stack-set --stack-set-name "$ss" --region ap-northeast-2 || true | |
| done | |
| continue-on-error: true | |
| # ── Linked account 1 ────────────────────────────────────────────────── | |
| - name: Assume role — linked account 1 | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_LINKED1_ACCOUNT_ID }}:role/GitHubActionsE2ERole | |
| aws-region: ap-northeast-2 | |
| continue-on-error: true | |
| - name: Guard — detect in-flight E2E (linked account 1) | |
| id: guard_l1 | |
| run: | | |
| if python3 .github/scripts/nightly_cleanup_guard.py check-account; then | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| fi | |
| continue-on-error: true | |
| - name: Delete tagged resources — linked account 1 | |
| if: steps.guard_l1.outputs.skip != 'true' | |
| working-directory: .github/scripts | |
| run: | | |
| # Sweep resources tagged with either the Lambda-applied | |
| # `map-migrated` key or the pre-applied `e2e-run-id` key (PR #7.a | |
| # switched E2E pre-tagging from map-migrated to e2e-run-id so | |
| # verify_tags actually tests the Lambda). Empty --tag-value means | |
| # "match any value" — covers every past run's per-run MPE/run-id tag. | |
| python3 teardown.py \ | |
| --all \ | |
| --sweep-keys map-migrated,e2e-run-id \ | |
| --regions ap-northeast-2,us-east-1,us-west-2 | |
| continue-on-error: true | |
| - name: Sweep orphaned map-auto-tagger IAM roles — linked account 1 | |
| if: steps.guard_l1.outputs.skip != 'true' | |
| run: python3 .github/scripts/sweep_iam_roles.py | |
| continue-on-error: true | |
| # ── Linked account 2 ────────────────────────────────────────────────── | |
| - name: Assume role — linked account 2 | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_LINKED2_ACCOUNT_ID }}:role/GitHubActionsE2ERole | |
| aws-region: ap-northeast-2 | |
| continue-on-error: true | |
| - name: Guard — detect in-flight E2E (linked account 2) | |
| id: guard_l2 | |
| run: | | |
| if python3 .github/scripts/nightly_cleanup_guard.py check-account; then | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| fi | |
| continue-on-error: true | |
| - name: Delete tagged resources — linked account 2 | |
| if: steps.guard_l2.outputs.skip != 'true' | |
| working-directory: .github/scripts | |
| run: | | |
| # Sweep resources tagged with either the Lambda-applied | |
| # `map-migrated` key or the pre-applied `e2e-run-id` key (PR #7.a | |
| # switched E2E pre-tagging from map-migrated to e2e-run-id so | |
| # verify_tags actually tests the Lambda). Empty --tag-value means | |
| # "match any value" — covers every past run's per-run MPE/run-id tag. | |
| python3 teardown.py \ | |
| --all \ | |
| --sweep-keys map-migrated,e2e-run-id \ | |
| --regions ap-northeast-2,us-east-1,us-west-2 | |
| continue-on-error: true | |
| - name: Sweep orphaned map-auto-tagger IAM roles — linked account 2 | |
| if: steps.guard_l2.outputs.skip != 'true' | |
| run: python3 .github/scripts/sweep_iam_roles.py | |
| continue-on-error: true | |
| # ── Linked account 3 ────────────────────────────────────────────────── | |
| - name: Assume role — linked account 3 | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_LINKED3_ACCOUNT_ID }}:role/GitHubActionsE2ERole | |
| aws-region: ap-northeast-2 | |
| continue-on-error: true | |
| - name: Guard — detect in-flight E2E (linked account 3) | |
| id: guard_l3 | |
| run: | | |
| if python3 .github/scripts/nightly_cleanup_guard.py check-account; then | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| fi | |
| continue-on-error: true | |
| - name: Delete tagged resources — linked account 3 | |
| if: steps.guard_l3.outputs.skip != 'true' | |
| working-directory: .github/scripts | |
| run: | | |
| # Sweep resources tagged with either the Lambda-applied | |
| # `map-migrated` key or the pre-applied `e2e-run-id` key (PR #7.a | |
| # switched E2E pre-tagging from map-migrated to e2e-run-id so | |
| # verify_tags actually tests the Lambda). Empty --tag-value means | |
| # "match any value" — covers every past run's per-run MPE/run-id tag. | |
| python3 teardown.py \ | |
| --all \ | |
| --sweep-keys map-migrated,e2e-run-id \ | |
| --regions ap-northeast-2,us-east-1,us-west-2 | |
| continue-on-error: true | |
| - name: Sweep orphaned map-auto-tagger IAM roles — linked account 3 | |
| if: steps.guard_l3.outputs.skip != 'true' | |
| run: python3 .github/scripts/sweep_iam_roles.py | |
| continue-on-error: true | |
| # ── Linked account 4 ────────────────────────────────────────────────── | |
| - name: Assume role — linked account 4 | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_LINKED4_ACCOUNT_ID }}:role/GitHubActionsE2ERole | |
| aws-region: ap-northeast-2 | |
| continue-on-error: true | |
| - name: Guard — detect in-flight E2E (linked account 4) | |
| id: guard_l4 | |
| run: | | |
| if python3 .github/scripts/nightly_cleanup_guard.py check-account; then | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| fi | |
| continue-on-error: true | |
| - name: Delete tagged resources — linked account 4 | |
| if: steps.guard_l4.outputs.skip != 'true' | |
| working-directory: .github/scripts | |
| run: | | |
| # Sweep resources tagged with either the Lambda-applied | |
| # `map-migrated` key or the pre-applied `e2e-run-id` key (PR #7.a | |
| # switched E2E pre-tagging from map-migrated to e2e-run-id so | |
| # verify_tags actually tests the Lambda). Empty --tag-value means | |
| # "match any value" — covers every past run's per-run MPE/run-id tag. | |
| python3 teardown.py \ | |
| --all \ | |
| --sweep-keys map-migrated,e2e-run-id \ | |
| --regions ap-northeast-2,us-east-1,us-west-2 | |
| continue-on-error: true | |
| - name: Sweep orphaned map-auto-tagger IAM roles — linked account 4 | |
| if: steps.guard_l4.outputs.skip != 'true' | |
| run: python3 .github/scripts/sweep_iam_roles.py | |
| continue-on-error: true | |
| # ── Linked account 5 ────────────────────────────────────────────────── | |
| - name: Assume role — linked account 5 | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_LINKED5_ACCOUNT_ID }}:role/GitHubActionsE2ERole | |
| aws-region: ap-northeast-2 | |
| continue-on-error: true | |
| - name: Guard — detect in-flight E2E (linked account 5) | |
| id: guard_l5 | |
| run: | | |
| if python3 .github/scripts/nightly_cleanup_guard.py check-account; then | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| fi | |
| continue-on-error: true | |
| - name: Delete tagged resources — linked account 5 | |
| if: steps.guard_l5.outputs.skip != 'true' | |
| working-directory: .github/scripts | |
| run: | | |
| # Sweep resources tagged with either the Lambda-applied | |
| # `map-migrated` key or the pre-applied `e2e-run-id` key (PR #7.a | |
| # switched E2E pre-tagging from map-migrated to e2e-run-id so | |
| # verify_tags actually tests the Lambda). Empty --tag-value means | |
| # "match any value" — covers every past run's per-run MPE/run-id tag. | |
| python3 teardown.py \ | |
| --all \ | |
| --sweep-keys map-migrated,e2e-run-id \ | |
| --regions ap-northeast-2,us-east-1,us-west-2 | |
| continue-on-error: true | |
| - name: Sweep orphaned map-auto-tagger IAM roles — linked account 5 | |
| if: steps.guard_l5.outputs.skip != 'true' | |
| run: python3 .github/scripts/sweep_iam_roles.py | |
| continue-on-error: true |