8317547: Enhance TLS connection support

Reviewed-by: mbalao, andrew
Backport-of: 066482f9686ca81068f9386322afda8e73323f5e

jdk/src/share/classes/com/sun/crypto/provider/RSACipher.java

@@ -98,6 +98,7 @@ public final class RSACipher extends CipherSpi {
 
     // cipher parameter for OAEP padding and TLS RSA premaster secret
     private AlgorithmParameterSpec spec = null;
+    private boolean forTlsPremasterSecret = false;
 
     // buffer for the data
     private byte[] buffer;
@@ -290,6 +291,7 @@ private void init(int opmode, Key key, SecureRandom random,
                 }
 
                 spec = params;
+                forTlsPremasterSecret = true;
                 this.random = random;   // for TLS RSA premaster secret
             }
             int blockType = (mode <= MODE_DECRYPT) ? RSAPadding.PAD_BLOCKTYPE_2
@@ -381,7 +383,7 @@ private byte[] doFinal() throws BadPaddingException,
                 byte[] decryptBuffer = RSACore.convert(buffer, 0, bufOfs);
                 paddingCopy = RSACore.rsa(decryptBuffer, privateKey, false);
                 result = padding.unpad(paddingCopy);
-                if (result == null) {
+                if (result == null && !forTlsPremasterSecret) {
                     throw new BadPaddingException
                             ("Padding error in decryption");
                 }
@@ -469,26 +471,22 @@ protected Key engineUnwrap(byte[] wrappedKey, String algorithm,
 
         boolean isTlsRsaPremasterSecret =
                 algorithm.equals("TlsRsaPremasterSecret");
-        Exception failover = null;
         byte[] encoded = null;
 
         update(wrappedKey, 0, wrappedKey.length);
         try {
             encoded = doFinal();
-        } catch (BadPaddingException e) {
-            if (isTlsRsaPremasterSecret) {
-                failover = e;
-            } else {
-                throw new InvalidKeyException("Unwrapping failed", e);
-            }
-        } catch (IllegalBlockSizeException e) {
-            // should not occur, handled with length check above
+        } catch (BadPaddingException | IllegalBlockSizeException e) {
+            // BadPaddingException cannot happen for TLS RSA unwrap.
+            // In that case, padding error is indicated by returning null.
+            // IllegalBlockSizeException cannot happen in any case,
+            // because of the length check above.
             throw new InvalidKeyException("Unwrapping failed", e);
         }
 
         try {
             if (isTlsRsaPremasterSecret) {
-                if (!(spec instanceof TlsRsaPremasterSecretParameterSpec)) {
+                if (!forTlsPremasterSecret) {
                     throw new IllegalStateException(
                             "No TlsRsaPremasterSecretParameterSpec specified");
                 }
@@ -497,7 +495,7 @@ protected Key engineUnwrap(byte[] wrappedKey, String algorithm,
                 encoded = KeyUtil.checkTlsPreMasterSecretKey(
                         ((TlsRsaPremasterSecretParameterSpec) spec).getClientVersion(),
                         ((TlsRsaPremasterSecretParameterSpec) spec).getServerVersion(),
-                        random, encoded, (failover != null));
+                        random, encoded, encoded == null);
             }
 
             return ConstructKeys.constructKey(encoded, algorithm, type);

jdk/src/share/classes/sun/security/util/KeyUtil.java

@@ -253,44 +253,53 @@ public static final boolean isOracleJCEProvider(String providerName) {
      *         contains the lower of that suggested by the client in the client
      *         hello and the highest supported by the server.
      * @param  encoded the encoded key in its "RAW" encoding format
-     * @param  isFailover whether or not the previous decryption of the
-     *         encrypted PreMasterSecret message run into problem
+     * @param  failure true if encoded is incorrect according to previous checks
      * @return the polished PreMasterSecret key in its "RAW" encoding format
      */
     public static byte[] checkTlsPreMasterSecretKey(
             int clientVersion, int serverVersion, SecureRandom random,
-            byte[] encoded, boolean isFailOver) {
+            byte[] encoded, boolean failure) {
+
+        byte[] tmp;
 
         if (random == null) {
             random = JCAUtil.getSecureRandom();
         }
         byte[] replacer = new byte[48];
         random.nextBytes(replacer);
 
-        if (!isFailOver && (encoded != null)) {
-            // check the length
-            if (encoded.length != 48) {
-                // private, don't need to clone the byte array.
-                return replacer;
-            }
-
-            int encodedVersion =
-                    ((encoded[0] & 0xFF) << 8) | (encoded[1] & 0xFF);
-            if (clientVersion != encodedVersion) {
-                if (clientVersion > 0x0301 ||               // 0x0301: TLSv1
-                       serverVersion != encodedVersion) {
-                    encoded = replacer;
-                }   // Otherwise, For compatibility, we maintain the behavior
-                    // that the version in pre_master_secret can be the
-                    // negotiated version for TLS v1.0 and SSL v3.0.
-            }
+        if (failure) {
+            tmp = replacer;
+        } else {
+            tmp = encoded;
+        }
 
+        if (tmp == null) {
+            encoded = replacer;
+        } else {
+            encoded = tmp;
+        }
+        // check the length
+        if (encoded.length != 48) {
             // private, don't need to clone the byte array.
-            return encoded;
+            tmp = replacer;
+        } else {
+            tmp = encoded;
         }
 
-        // private, don't need to clone the byte array.
-        return replacer;
+        int encodedVersion =
+                ((tmp[0] & 0xFF) << 8) | (tmp[1] & 0xFF);
+        int check1 = 0;
+        int check2 = 0;
+        int check3 = 0;
+        if (clientVersion != encodedVersion) check1 = 1;
+        if (clientVersion > 0x0301) check2 = 1;
+        if (serverVersion != encodedVersion) check3 = 1;
+        if ((check1 & (check2 | check3)) == 1) {
+            return replacer;
+        } else {
+            return tmp;
+        }
     }
 
     /**

jdk/src/windows/classes/sun/security/mscapi/CRSACipher.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,7 @@
 import java.security.Key;
 import java.security.interfaces.*;
 import java.security.spec.*;
+import java.util.Arrays;
 
 import javax.crypto.*;
 import javax.crypto.spec.*;
@@ -61,6 +62,9 @@
  */
 public final class CRSACipher extends CipherSpi {
 
+    private static final int ERROR_INVALID_PARAMETER = 0x57;
+    private static final int NTE_INVALID_PARAMETER = 0x80090027;
+
     // constant for an empty byte array
     private final static byte[] B0 = new byte[0];
 
@@ -101,6 +105,8 @@ public final class CRSACipher extends CipherSpi {
     // cipher parameter for TLS RSA premaster secret
     private AlgorithmParameterSpec spec = null;
 
+    private boolean forTlsPremasterSecret = false;
+
     // the source of randomness
     private SecureRandom random;
 
@@ -171,6 +177,9 @@ protected void engineInit(int opmode, Key key,
             }
             spec = params;
             this.random = random;   // for TLS RSA premaster secret
+            this.forTlsPremasterSecret = true;
+        } else {
+            this.forTlsPremasterSecret = false;
         }
         init(opmode, key);
     }
@@ -277,8 +286,7 @@ private void update(byte[] in, int inOfs, int inLen) {
     }
 
     // internal doFinal() method. Here we perform the actual RSA operation
-    private byte[] doFinal() throws BadPaddingException,
-            IllegalBlockSizeException {
+    private byte[] doFinal() throws IllegalBlockSizeException {
         if (bufOfs > buffer.length) {
             throw new IllegalBlockSizeException("Data must not be longer "
                 + "than " + (buffer.length - paddingLength)  + " bytes");
@@ -307,7 +315,7 @@ private byte[] doFinal() throws BadPaddingException,
                 throw new AssertionError("Internal error");
             }
 
-        } catch (KeyException e) {
+        } catch (KeyException | BadPaddingException e) {
             throw new ProviderException(e);
 
         } finally {
@@ -330,14 +338,14 @@ protected int engineUpdate(byte[] in, int inOfs, int inLen, byte[] out,
 
     // see JCE spec
     protected byte[] engineDoFinal(byte[] in, int inOfs, int inLen)
-            throws BadPaddingException, IllegalBlockSizeException {
+            throws IllegalBlockSizeException {
         update(in, inOfs, inLen);
         return doFinal();
     }
 
     // see JCE spec
     protected int engineDoFinal(byte[] in, int inOfs, int inLen, byte[] out,
-            int outOfs) throws ShortBufferException, BadPaddingException,
+            int outOfs) throws ShortBufferException,
             IllegalBlockSizeException {
         if (outputSize > out.length - outOfs) {
             throw new ShortBufferException
@@ -353,6 +361,7 @@ protected int engineDoFinal(byte[] in, int inOfs, int inLen, byte[] out,
     // see JCE spec
     protected byte[] engineWrap(Key key) throws InvalidKeyException,
             IllegalBlockSizeException {
+
         byte[] encoded = key.getEncoded(); // TODO - unextractable key
         if ((encoded == null) || (encoded.length == 0)) {
             throw new InvalidKeyException("Could not obtain encoded key");
@@ -361,12 +370,7 @@ protected byte[] engineWrap(Key key) throws InvalidKeyException,
             throw new InvalidKeyException("Key is too long for wrapping");
         }
         update(encoded, 0, encoded.length);
-        try {
-            return doFinal();
-        } catch (BadPaddingException e) {
-            // should not occur
-            throw new InvalidKeyException("Wrapping failed", e);
-        }
+        return doFinal();
     }
 
     // see JCE spec
@@ -387,31 +391,31 @@ protected java.security.Key engineUnwrap(byte[] wrappedKey,
         update(wrappedKey, 0, wrappedKey.length);
         try {
             encoded = doFinal();
-        } catch (BadPaddingException e) {
-            if (isTlsRsaPremasterSecret) {
-                failover = e;
-            } else {
-                throw new InvalidKeyException("Unwrapping failed", e);
-            }
         } catch (IllegalBlockSizeException e) {
             // should not occur, handled with length check above
             throw new InvalidKeyException("Unwrapping failed", e);
         }
 
-        if (isTlsRsaPremasterSecret) {
-            if (!(spec instanceof TlsRsaPremasterSecretParameterSpec)) {
-                throw new IllegalStateException(
-                        "No TlsRsaPremasterSecretParameterSpec specified");
+        try {
+            if (isTlsRsaPremasterSecret) {
+                if (!forTlsPremasterSecret) {
+                    throw new IllegalStateException(
+                            "No TlsRsaPremasterSecretParameterSpec specified");
+                }
+
+                // polish the TLS premaster secret
+                encoded = KeyUtil.checkTlsPreMasterSecretKey(
+                        ((TlsRsaPremasterSecretParameterSpec) spec).getClientVersion(),
+                        ((TlsRsaPremasterSecretParameterSpec) spec).getServerVersion(),
+                        random, encoded, encoded == null);
             }
 
-            // polish the TLS premaster secret
-            encoded = KeyUtil.checkTlsPreMasterSecretKey(
-                ((TlsRsaPremasterSecretParameterSpec)spec).getClientVersion(),
-                ((TlsRsaPremasterSecretParameterSpec)spec).getServerVersion(),
-                random, encoded, (failover != null));
+            return constructKey(encoded, algorithm, type);
+        } finally {
+            if (encoded != null) {
+                Arrays.fill(encoded, (byte) 0);
+            }
         }
-
-        return constructKey(encoded, algorithm, type);
     }
 
     // see JCE spec
@@ -495,7 +499,23 @@ private static Key constructKey(byte[] encodedKey,
      * Encrypt/decrypt a data buffer using Microsoft Crypto API with HCRYPTKEY.
      * It expects and returns ciphertext data in big-endian form.
      */
-    private native static byte[] encryptDecrypt(byte[] data, int dataSize,
-        long hCryptKey, boolean doEncrypt) throws KeyException;
+    private byte[] encryptDecrypt(byte[] data, int dataSize,
+        long hCryptKey, boolean doEncrypt) throws KeyException, BadPaddingException {
+        int[] returnStatus = new int[1];
+        byte[] result= encryptDecrypt(returnStatus, data, dataSize, hCryptKey, doEncrypt);
+        if ((returnStatus[0] == ERROR_INVALID_PARAMETER) || (returnStatus[0] == NTE_INVALID_PARAMETER)) {
+            if (forTlsPremasterSecret) {
+                result = null;
+            } else {
+                throw new BadPaddingException("Error " + returnStatus[0] + " returned by MSCAPI");
+            }
+        } else if (returnStatus[0] != 0) {
+            throw new KeyException("Error " + returnStatus[0] + " returned by MSCAPI");
+        }
+
+        return result;
+    }
+    private static native byte[] encryptDecrypt(int[] returnStatus, byte[] data, int dataSize,
+            long key, boolean doEncrypt) throws KeyException;
 
 }

jdk/src/windows/native/sun/security/mscapi/security.cpp

@@ -1832,18 +1832,25 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_CKeyStore_destroyKeyContainer
 /*
  * Class:     sun_security_mscapi_CRSACipher
  * Method:    encryptDecrypt
- * Signature: ([BIJZ)[B
+ * Signature: ([I[BIJZ)[B
  */
 JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_encryptDecrypt
-  (JNIEnv *env, jclass clazz, jbyteArray jData, jint jDataSize, jlong hKey,
+  (JNIEnv *env, jclass clazz, jintArray jResultStatus, jbyteArray jData, jint jDataSize, jlong hKey,
    jboolean doEncrypt)
 {
     jbyteArray result = NULL;
     jbyte* pData = NULL;
+    jbyte* resultData = NULL;
     DWORD dwDataLen = jDataSize;
     DWORD dwBufLen = env->GetArrayLength(jData);
     DWORD i;
     BYTE tmp;
+    BOOL success;
+    DWORD ss = ERROR_SUCCESS;
+    DWORD lastError = ERROR_SUCCESS;
+    DWORD resultLen = 0;
+    DWORD pmsLen = 48;
+    jbyte pmsArr[48] = {0};
 
     __try
     {
@@ -1870,6 +1877,8 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_encryptDecrypt
                 pData[i] = pData[dwBufLen - i -1];
                 pData[dwBufLen - i - 1] = tmp;
             }
+            resultData = pData;
+            resultLen = dwBufLen;
         } else {
             // convert to little-endian
             for (i = 0; i < dwBufLen / 2; i++) {
@@ -1879,21 +1888,28 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_encryptDecrypt
             }
 
             // decrypt
-            if (! ::CryptDecrypt((HCRYPTKEY) hKey, 0, TRUE, 0, (BYTE *)pData, //deprecated
-                &dwBufLen)) {
-
-                ThrowException(env, KEY_EXCEPTION, GetLastError());
-                __leave;
+            success = ::CryptDecrypt((HCRYPTKEY) hKey, 0, TRUE, 0, (BYTE *)pData, //deprecated
+                &dwBufLen);
+            lastError = GetLastError();
+            if (success) {
+                ss = ERROR_SUCCESS;
+                resultData = pData;
+                resultLen = dwBufLen;
+            } else {
+                ss = lastError;
+                resultData = pmsArr;
+                resultLen = pmsLen;
             }
+            env->SetIntArrayRegion(jResultStatus, 0, 1, (jint*) &ss);
         }
 
-        // Create new byte array
-        if ((result = env->NewByteArray(dwBufLen)) == NULL) {
+            // Create new byte array
+        if ((result = env->NewByteArray(resultLen)) == NULL) {
             __leave;
         }
 
         // Copy data from native buffer to Java buffer
-        env->SetByteArrayRegion(result, 0, dwBufLen, (jbyte*) pData);
+        env->SetByteArrayRegion(result, 0, resultLen, (jbyte*) resultData);
     }
     __finally
     {