diff --git a/oauthenticator/generic.py b/oauthenticator/generic.py
index b3166a90..f29c8a54 100644
--- a/oauthenticator/generic.py
+++ b/oauthenticator/generic.py
@@ -116,7 +116,7 @@ def get_user_groups(self, user_info):
 
     async def user_is_authorized(self, auth_model):
         user_info = auth_model["auth_state"][self.user_auth_state_key]
-        if self.allowed_groups:
+        if not self.allowed_users and (self.allowed_groups or self.admin_groups):
             self.log.info(
                 f"Validating if user claim groups match any of {self.allowed_groups}"
             )
diff --git a/oauthenticator/google.py b/oauthenticator/google.py
index 00b2dd9c..b6afa520 100644
--- a/oauthenticator/google.py
+++ b/oauthenticator/google.py
@@ -135,7 +135,7 @@ async def user_is_authorized(self, auth_model):
                     403, f"Google account domain @{user_email_domain} not authorized."
                 )
 
-        if self.allowed_google_groups:
+        if not self.allowed_users and (self.allowed_google_groups or self.admin_google_groups):
             google_groups = self._google_groups_for_user(user_email, user_email_domain)
             if not google_groups:
                 return False
diff --git a/oauthenticator/openshift.py b/oauthenticator/openshift.py
index d0812271..45b2330d 100644
--- a/oauthenticator/openshift.py
+++ b/oauthenticator/openshift.py
@@ -115,7 +115,7 @@ async def user_is_authorized(self, auth_model):
         user_groups = set(auth_model['auth_state']['openshift_user']['groups'])
         username = auth_model['name']
 
-        if self.allowed_groups or self.admin_groups:
+        if not self.allowed_users and (self.allowed_groups or self.admin_groups):
             msg = f"username:{username} User not in any of the allowed/admin groups"
             # User is authorized if either in allowed_groups or in admin_groups
             if not self.user_groups_in_allowed_groups(