diff --git a/dist/index.js b/dist/index.js
index 6610c8a..41ce623 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -36944,15 +36944,17 @@ try {
         // Temp environments enabled
         if (github.context.eventName === "pull_request") {
           const prEvent = github.context.payload;
-          const targetLabel = value.label;
-          if (!targetLabel) continue;
-          const hasLabel =
-            (prEvent.action === "labeled" &&
-              prEvent.label.name === targetLabel) ||
-            prEvent.pull_request.labels.some(
-              (label) => label.name === targetLabel,
+          let matches = false;
+          if (value.label) {
+            matches = prEvent.pull_request.labels.some(
+              (label) => label.name === value.label,
             );
-          if (hasLabel) {
+          } else if (value.removed_label) {
+            matches = !prEvent.pull_request.labels.some(
+              (label) => label.name === value.removed_label,
+            );
+          }
+          if (matches) {
             patterns = ref;
             value.name = `${value.name || toTitleCase(env)} #${prEvent.number}`;
             env += prEvent.number;
diff --git a/src/index.js b/src/index.js
index 60d5f0d..4cae47e 100644
--- a/src/index.js
+++ b/src/index.js
@@ -31,15 +31,17 @@ try {
         // Temp environments enabled
         if (github.context.eventName === "pull_request") {
           const prEvent = github.context.payload;
-          const targetLabel = value.label;
-          if (!targetLabel) continue;
-          const hasLabel =
-            (prEvent.action === "labeled" &&
-              prEvent.label.name === targetLabel) ||
-            prEvent.pull_request.labels.some(
-              (label) => label.name === targetLabel,
+          let matches = false;
+          if (value.label) {
+            matches = prEvent.pull_request.labels.some(
+              (label) => label.name === value.label,
             );
-          if (hasLabel) {
+          } else if (value.removed_label) {
+            matches = !prEvent.pull_request.labels.some(
+              (label) => label.name === value.removed_label,
+            );
+          }
+          if (matches) {
             patterns = ref;
             value.name = `${value.name || toTitleCase(env)} #${prEvent.number}`;
             env += prEvent.number;