Bug 39116277 - [38668667->25.09] Extend TCP Proxy TcpProcessor thread Spinning on 0-Byte Read on Closed Sockets causing High CPU

Bug 38668667 - Extend TCP Proxy TcpProcessor thread Spinning on 0-Byte Read on Closed Sockets causing High CPU.

Issue:
- When a client sends a partial TLS record and closes the connection, the server receives incomplete encrypted data.
- SSLEngine.unwrap() cannot decrypt the record and returns BUFFER_UNDERFLOW because more bytes are required.
- Since the socket has already reached EOS, no additional data can arrive to complete the TLS record.
- The remaining encrypted bytes may keep the channel marked readable, causing the connection to be processed repeatedly.   

Fix:
- Updated the read logic to return -1 when the socket has reached EOS, no bytes were produced for the caller buffer (cbFill == 0), and no decrypted bytes remain in the clear buffer.
- The code also checks that fillClearBuffer() returns 0, ensuring that no additional bytes can be decrypted from the remaining encrypted data.
- This indicates that the remaining data if present in Encyrpted Buffer is likely a partial TLS record that cannot be completed.
- Returning -1 signals end-of-stream and allows the TcpProcessor to close the connection, preventing repeated processing of the same socket.

[git-p4: depot-paths = "//dev/coherence-ce/main/": change = 119310]

Author: sunny-gali

prj/coherence-core/src/main/java/com/oracle/coherence/common/internal/net/ssl/SSLSocketChannel.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * https://oss.oracle.com/licenses/upl.
@@ -436,15 +436,13 @@ protected long readInternal(ByteBuffer[] aBuffDst, int ofBuff, int cBuff)
         // COH-21678 - check both clear and encrypted as the SSL engine may consume the decrypt resulting in an empty clear buffer
         boolean fMore = f_buffClearIn.hasRemaining() || fillClearBuffer() > 0 || f_buffEncIn.hasRemaining();
         markKeysReadable(fMore);
-
-        if (cbFill == 0 && cbRead == -1 && !fMore)
-            {
-            return -1;
-            }
-        else
-            {
-            return cbFill;
-            }
+        // COH-33073 - Prevent TcpProcessor spinning on closed sockets.
+        //       If no bytes were written to the destination buffer and the socket has reached EOS,
+        //       check fillClearBuffer() to ensure any remaining encrypted data cannot still be decrypted.
+        //       If it produces no bytes and the clear buffer is empty, the remaining data is likely a
+        //       partial TLS record that cannot be completed, so return -1 and allow TcpProcessor to close
+        //       the connection.
+        return (cbFill == 0 && cbRead == -1 && !(f_buffClearIn.hasRemaining()) && fillClearBuffer() == 0) ? -1 : cbFill;
         }
 
 

prj/test/functional/extend/src/main/java/extend/DistExtendDirectSSLTests.java

@@ -1,16 +1,28 @@
 /*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * https://oss.oracle.com/licenses/upl.
  */
 
 package extend;
 
+import com.oracle.bedrock.runtime.coherence.CoherenceClusterMember;
+import com.oracle.bedrock.testsupport.deferred.Eventually;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.util.Properties;
 
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Test;
 
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertTrue;
 
 /**
 * A collection of functional tests for Coherence*Extend that use the
@@ -38,9 +50,10 @@ public DistExtendDirectSSLTests()
     * Initialize the test class.
     */
     @BeforeClass
-    public static void startup()
+    public static void _startup()
         {
-        startCacheServerWithProxy("DistExtendDirectSSLTests", "extend", "server-cache-config-ssl.xml");
+        setupProps();
+        s_member = startCacheServerWithProxy("DistExtendDirectSSLTests", "extend", "server-cache-config-ssl.xml");
         }
 
     /**
@@ -51,4 +64,81 @@ public static void shutdown()
         {
         stopCacheServer("DistExtendDirectSSLTests");
         }
+
+    /**
+     * COH-33073 - ensure SSL-configured ExtendTcpProxy (TcpProcessor path)
+     * cleanly closes a connection when it receives malformed TLS bytes
+     * that cause BUFFER_UNDERFLOW / SSL decode failure.
+     */
+    @Test
+    public void testProxyClosesConnectionOnIncompleteTlsRecord()
+            throws Exception
+        {
+        Eventually.assertDeferred(() -> s_member.isServiceRunning("ExtendTcpProxyService"), is(true));
+
+        Properties props = System.getProperties();
+        String     sHost = props.getProperty("test.extend.address.remote", "127.0.0.1");
+        int        nPort = Integer.parseInt(props.getProperty("test.extend.port"));
+
+        byte[] abPayload = new byte[]
+            {
+            (byte) 0x16, (byte) 0x03, (byte) 0x03, (byte) 0x00,
+            (byte) 0x50, (byte) 0x01, (byte) 0x02, (byte) 0x03
+            };
+
+        boolean fClosed = false;
+
+        try (Socket socket = new Socket(sHost, nPort))
+            {
+            socket.setSoTimeout(2000);
+
+            OutputStream out = socket.getOutputStream();
+            out.write(abPayload);
+            out.flush();
+
+            // half close client -> server
+            socket.shutdownOutput();
+
+            InputStream in     = socket.getInputStream();
+            byte[]      buffer = new byte[32];
+
+            int attempts = 3;
+
+            while (attempts-- > 0)
+                {
+                try
+                    {
+                    int n = in.read(buffer);
+                    if (n == -1)
+                        {
+                        // graceful close (FIN)
+                        System.out.println("Server closed socket (FIN) after malformed TLS payload");
+                        fClosed = true;
+                        break;
+                        }
+                    continue;
+                    }
+                catch (SocketException e)
+                    {
+                    // connection reset (RST) also means server closed the connection
+                    System.out.println("Server reset socket (RST) after malformed TLS payload: " + e.getMessage());
+                    fClosed = true;
+                    break;
+                    }
+                catch (SocketTimeoutException e)
+                    {
+                    // server kept connection open
+                    continue;
+                    }
+                }
+            }
+
+        assertTrue("ExtendTcpProxyService did not close the connection for malformed TLS input", fClosed);
+
+        Eventually.assertDeferred(() -> s_member.isServiceRunning("ExtendTcpProxyService"), is(true));
+        }
+
+    // ----- data members ---------------------------------------------------
+
+    private static CoherenceClusterMember s_member;
     }