JAVA-5949 preserve connection pool on backpressure errors when establishing connections (#1900)

Author: nhachicha

driver-core/src/main/com/mongodb/internal/connection/BackpressureErrorLabeler.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright 2008-present MongoDB, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.mongodb.internal.connection;
+
+import com.mongodb.MongoException;
+import com.mongodb.MongoSocketException;
+
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import javax.net.ssl.SSLProtocolException;
+import java.net.UnknownHostException;
+import java.security.cert.CertPathBuilderException;
+import java.security.cert.CertPathValidatorException;
+import java.security.cert.CertificateException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+
+/**
+ * Attaches {@link MongoException#SYSTEM_OVERLOADED_ERROR_LABEL} and
+ * {@link MongoException#RETRYABLE_ERROR_LABEL} to network errors encountered during connection
+ * establishment or the hello message, per the CMAP specification.
+ *
+ * <p>This is topology-agnostic: it must be invoked from the connection-establishment path so that
+ * both default SDAM and load-balanced modes are covered.
+ */
+final class BackpressureErrorLabeler {
+
+    /**
+     * BouncyCastle TLS fatal-alert exception type names.
+     */
+    private static final Set<String> BOUNCY_CASTLE_TLS_FATAL_TYPE_NAMES = Collections.unmodifiableSet(
+            new HashSet<>(Arrays.asList(
+                    "org.bouncycastle.tls.TlsFatalAlert",
+                    "org.bouncycastle.tls.TlsFatalAlertReceived",
+                    "org.bouncycastle.tls.crypto.TlsCryptoException")));
+
+    /**
+     * RFC 5246 / RFC 8446 alert descriptions that surface in BouncyCastle TLS exception messages.
+     * See <a href="https://github.com/bcgit/bc-java/blob/main/tls/src/main/java/org/bouncycastle/tls/AlertDescription.java">AlertDescription.java</a>.
+     * See <a href="https://datatracker.ietf.org/doc/html/rfc5246#section-7.2">(TLS) Protocol Version 1.2 - Alert Protocol</a>.
+     */
+    private static final Set<String> BOUNCY_CASTLE_TLS_ALERT_DESCRIPTIONS = Collections.unmodifiableSet(
+            new HashSet<>(Arrays.asList(
+                    "close_notify", "unexpected_message", "bad_record_mac", "decryption_failed",
+                    "record_overflow", "decompression_failure", "handshake_failure", "no_certificate",
+                    "bad_certificate", "unsupported_certificate", "certificate_revoked", "certificate_expired",
+                    "certificate_unknown", "illegal_parameter", "unknown_ca", "access_denied",
+                    "decode_error", "decrypt_error", "export_restriction", "protocol_version",
+                    "insufficient_security", "internal_error", "no_renegotiation", "unsupported_extension",
+                    "certificate_unobtainable", "unrecognized_name", "bad_certificate_status_response",
+                    "bad_certificate_hash_value", "unknown_psk_identity", "no_application_protocol",
+                    "inappropriate_fallback", "missing_extension", "certificate_required")));
+
+    private BackpressureErrorLabeler() {
+    }
+
+    static void applyLabelsIfEligible(final Throwable t) {
+        if (!(t instanceof MongoSocketException)) {
+            return;
+        }
+        MongoSocketException socketException = (MongoSocketException) t;
+        if (isDnsLookupFailure(socketException)) {
+            return;
+        }
+        if (isTlsConfigurationError(socketException)) {
+            return;
+        }
+        // TODO-BACKPRESSURE Nabil - Add SOCKS5 check once JAVA-6194 is introduced
+        // async proxy error surfaces can be handled together — likely via a dedicated internal
+        // exception thrown from the proxy code path.
+        socketException.addLabel(MongoException.SYSTEM_OVERLOADED_ERROR_LABEL);
+        socketException.addLabel(MongoException.RETRYABLE_ERROR_LABEL);
+    }
+
+    private static boolean isDnsLookupFailure(final MongoSocketException t) {
+        Throwable cause = t.getCause();
+        while (cause != null) {
+            if (cause instanceof UnknownHostException) {
+                return true;
+            }
+            cause = cause.getCause();
+        }
+        return false;
+    }
+
+    private static boolean isTlsConfigurationError(final MongoSocketException t) {
+        Throwable cause = t.getCause();
+        while (cause != null) {
+            if (cause instanceof CertificateException
+                    || cause instanceof CertPathBuilderException
+                    || cause instanceof CertPathValidatorException
+                    || cause instanceof SSLPeerUnverifiedException
+                    || cause instanceof SSLProtocolException) {
+                return true;
+            }
+            if (cause instanceof SSLHandshakeException) {
+                String message = cause.getMessage();
+                if (message != null) {
+                    String lowerMessage = message.toLowerCase(Locale.ROOT);
+                    if (lowerMessage.contains("verify")
+                            || lowerMessage.contains("protocol")
+                            || lowerMessage.contains("cipher")
+                            || lowerMessage.contains("received fatal alert")) {
+                        return true;
+                    }
+                }
+            }
+            if (isBouncyCastleTlsError(cause)) {
+                return true;
+            }
+
+            cause = cause.getCause();
+        }
+        return false;
+    }
+
+    private static boolean isBouncyCastleTlsError(final Throwable cause) {
+        if (!isBouncyCastleTlsFatalType(cause.getClass())) {
+            return false;
+        }
+        String message = cause.getMessage();
+        if (message == null) {
+            return false;
+        }
+        String description = message.toLowerCase(Locale.ROOT);
+        for (String alertName : BOUNCY_CASTLE_TLS_ALERT_DESCRIPTIONS) {
+            if (description.contains(alertName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Walks the class hierarchy comparing fully qualified names so a subclass of a known BC type
+     * still matches.
+     */
+    private static boolean isBouncyCastleTlsFatalType(final Class<?> exceptionClass) {
+        Class<?> cls = exceptionClass;
+        while (cls != null) {
+            if (BOUNCY_CASTLE_TLS_FATAL_TYPE_NAMES.contains(cls.getName())) {
+                return true;
+            }
+            cls = cls.getSuperclass();
+        }
+        return false;
+    }
+}

driver-core/src/main/com/mongodb/internal/connection/DefaultSdamServerDescriptionManager.java

@@ -137,6 +137,9 @@ private void handleException(final SdamIssue sdamIssue, final boolean beforeHand
             serverMonitor.connect();
         } else if (sdamIssue.relatedToNetworkNotTimeout()
                 || (beforeHandshake && (sdamIssue.relatedToNetworkTimeout() || sdamIssue.relatedToAuth()))) {
+            if (sdamIssue.hasSystemOverloadedLabel()) {
+                return;
+            }
             updateDescription(sdamIssue.serverDescription());
             connectionPool.invalidate(sdamIssue.exception().orElse(null));
             serverMonitor.cancelCurrentCheck();

driver-core/src/main/com/mongodb/internal/connection/InternalStreamConnection.java

@@ -229,9 +229,11 @@ public void open(final OperationContext originalOperationContext) {
         isTrue("Open already called", stream == null);
         stream = streamFactory.create(serverId.getAddress());
         OperationContext operationContext = originalOperationContext;
+        boolean beforeHandshake = true;
         try {
             stream.open(operationContext);
             InternalConnectionInitializationDescription initializationDescription = connectionInitializer.startHandshake(this, operationContext);
+            beforeHandshake = false;
 
             operationContext = operationContext.withOverride(TimeoutContext::withNewlyStartedMaintenanceTimeout);
             initAfterHandshakeStart(initializationDescription);
@@ -240,6 +242,9 @@ public void open(final OperationContext originalOperationContext) {
             initAfterHandshakeFinish(initializationDescription);
         } catch (Throwable t) {
             close();
+            if (beforeHandshake) {
+                BackpressureErrorLabeler.applyLabelsIfEligible(t);
+            }
             if (t instanceof MongoException) {
                 throw (MongoException) t;
             } else {
@@ -262,6 +267,7 @@ public void completed(@Nullable final Void aVoid) {
                             (initialResult, initialException) -> {
                                     if (initialException != null) {
                                         close();
+                                        BackpressureErrorLabeler.applyLabelsIfEligible(initialException);
                                         callback.onResult(null, initialException);
                                     } else {
                                         assertNotNull(initialResult);
@@ -277,11 +283,13 @@ public void completed(@Nullable final Void aVoid) {
                 @Override
                 public void failed(final Throwable t) {
                     close();
+                    BackpressureErrorLabeler.applyLabelsIfEligible(t);
                     callback.onResult(null, t);
                 }
             });
         } catch (Throwable t) {
             close();
+            BackpressureErrorLabeler.applyLabelsIfEligible(t);
             callback.onResult(null, t);
         }
     }

driver-core/src/main/com/mongodb/internal/connection/SdamServerDescriptionManager.java

@@ -17,6 +17,7 @@
 package com.mongodb.internal.connection;
 
 import com.mongodb.MongoCommandException;
+import com.mongodb.MongoException;
 import com.mongodb.MongoNodeIsRecoveringException;
 import com.mongodb.MongoNotPrimaryException;
 import com.mongodb.MongoSecurityException;
@@ -162,6 +163,11 @@ boolean relatedToWriteConcern() {
             return exception instanceof MongoWriteConcernWithResponseException;
         }
 
+        boolean hasSystemOverloadedLabel() {
+            return exception instanceof MongoException
+                    && ((MongoException) exception).hasErrorLabel(MongoException.SYSTEM_OVERLOADED_ERROR_LABEL);
+        }
+
         private static boolean stale(@Nullable final Throwable t, final ServerDescription currentServerDescription) {
             return TopologyVersionHelper.topologyVersion(t)
                     .map(candidateTopologyVersion -> TopologyVersionHelper.newerOrEqual(

driver-core/src/test/unit/com/mongodb/internal/connection/AbstractServerDiscoveryAndMonitoringTest.java

@@ -113,6 +113,7 @@ protected void applyApplicationError(final BsonDocument applicationError) {
 
         switch (when) {
             case "beforeHandshakeCompletes":
+                BackpressureErrorLabeler.applyLabelsIfEligible(exception);
                 server.sdamServerDescriptionManager().handleExceptionBeforeHandshake(
                         SdamIssue.of(exception, new SdamIssue.Context(server.serverId(), errorGeneration, maxWireVersion)));
                 break;