diff --git a/.github/mergify.yml b/.github/mergify.yml
index 82e632b700f5..68884a664767 100644
--- a/.github/mergify.yml
+++ b/.github/mergify.yml
@@ -14,9 +14,12 @@ pull_request_rules:
           # Important Notes
           {{ notes }}
           {%- endif -%}
-  - name: Automatic PR update
+  - name: One time automatic PR update
     conditions:
       - or:
           - "label=CI: Keep up to date"
     actions:
       update:
+      label:
+        remove:
+          - "CI: Keep up to date"