Bug 1461: Support for non-ASCII passphrases to client certificate files

https://winscp.net/tracker/1461

Source commit: aaab80f16566ef2d7153eb745517e70b6a07c494

libs/openssl/Makefile

@@ -15,7 +15,7 @@
 
 # Set your compiler options
 CC=bcc32
-CFLAG=-DWIN32_LEAN_AND_MEAN -q -w-ccc -w-rch -w-pia -w-aus -w-par -w-inl -w-eff -w-sus -w-dup -w-spa -w-csu -w-rvl  -c -tWC -tWM -DOPENSSL_SYSNAME_WIN32 -DL_ENDIAN -DDSO_WIN32 -D_stricmp=stricmp -D_strnicmp=strnicmp -O2 -ff -fp -DBN_ASM -DMD5_ASM -DSHA1_ASM -DOPENSSL_NO_CAMELLIA -DOPENSSL_NO_SEED -DOPENSSL_NO_RC5 -DOPENSSL_NO_MDC2 -DOPENSSL_NO_CAPIENG -DOPENSSL_NO_KRB5 -DOPENSSL_NO_ENGINE -DOPENSSL_NO_DYNAMIC_ENGINE -DOPENSSL_DISABLE_OLD_DES_SUPPORT -DNO_CHMOD -DOPENSSL_NO_DGRAM -DDOPENSSL_NO_EC_NISTP_64_GCC_128 -DOPENSSL_NO_WHIRLPOOL
+CFLAG=-DWIN32_LEAN_AND_MEAN -q -w-ccc -w-rch -w-pia -w-aus -w-par -w-inl -w-eff -w-sus -w-dup -w-spa -w-csu -w-rvl  -c -tWC -tWM -DOPENSSL_SYSNAME_WIN32 -DL_ENDIAN -DDSO_WIN32 -D_stricmp=stricmp -D_strnicmp=strnicmp -O2 -ff -fp -DBN_ASM -DMD5_ASM -DSHA1_ASM -DOPENSSL_NO_CAMELLIA -DOPENSSL_NO_SEED -DOPENSSL_NO_RC5 -DOPENSSL_NO_MDC2 -DOPENSSL_NO_CAPIENG -DOPENSSL_NO_KRB5 -DOPENSSL_NO_ENGINE -DOPENSSL_NO_DYNAMIC_ENGINE -DOPENSSL_DISABLE_OLD_DES_SUPPORT -DNO_CHMOD -DOPENSSL_NO_DGRAM -DDOPENSSL_NO_EC_NISTP_64_GCC_128 -DOPENSSL_NO_WHIRLPOOL -DPBE_UNICODE -DWINSCP
 LIB_CFLAG=
 
 # The OpenSSL directory

libs/openssl/crypto/evp/evp_pbe.c

@@ -171,7 +171,15 @@ int EVP_PBE_CipherInit(ASN1_OBJECT *pbe_obj, const char *pass, int passlen,
     if (!pass)
         passlen = 0;
     else if (passlen == -1)
+    {
+    #if defined(WINSCP) && defined(PBE_UNICODE)
+        // OPENSSL_asc2uni adds the trailing \0 to the length,
+        // even if input ascii password length does not include it
+        passlen = (wcslen((const wchar_t*)pass) * sizeof(wchar_t)) + sizeof(wchar_t);
+    #else
         passlen = strlen(pass);
+    #endif
+    }
 
     if (cipher_nid == -1)
         cipher = NULL;

libs/openssl/crypto/pkcs12/p12_kiss.c

@@ -106,7 +106,12 @@ int PKCS12_parse(PKCS12 *p12, const char *pass, EVP_PKEY **pkey, X509 **cert,
      * password are two different things...
      */
 
-    if (!pass || !*pass) {
+    if (!pass ||
+        (!pass[0]
+        #if defined(WINSCP) && defined(PBE_UNICODE)
+        && !pass[1]
+        #endif
+        )) {
         if (PKCS12_verify_mac(p12, NULL, 0))
             pass = NULL;
         else if (PKCS12_verify_mac(p12, "", 0))

libs/openssl/crypto/pkcs12/p12_mutl.c

@@ -93,7 +93,16 @@ int PKCS12_gen_mac(PKCS12 *p12, const char *pass, int passlen,
     md_size = EVP_MD_size(md_type);
     if (md_size < 0)
         return 0;
-    if (!PKCS12_key_gen(pass, passlen, salt, saltlen, PKCS12_MAC_ID, iter,
+    #if defined(WINSCP) && defined(PBE_UNICODE)
+    if (passlen < 0)
+    {
+      // PKCS12_key_gen_uni cannot handle -1 length (contrary to PKCS12_key_gen_asc).
+      // OPENSSL_asc2uni adds the trailing \0 to the length,
+      // even if input ascii password length does not include it.
+      passlen = (wcslen((wchar_t*)pass) * sizeof(wchar_t)) + sizeof(wchar_t);
+    }
+    #endif
+    if (!PKCS12_key_gen(pass, passlen, salt, saltlen, PKCS12_MAC_ID, iter,
                         md_size, key, md_type)) {
         PKCS12err(PKCS12_F_PKCS12_GEN_MAC, PKCS12_R_KEY_GEN_ERROR);
         return 0;

source/core/Common.cpp

@@ -2817,9 +2817,17 @@ void __fastcall ParseCertificate(const UnicodeString & Path,
 
   if (Pkcs12 != NULL)
   {
-    // Not sure about the UTF-8 encoding, but there's no wchar_t API
+    // Modeled after OPENSSL_asc2uni (reversed bitness to what UnicodeString/wchar_t use)
+    std::vector<char> Buf;
+    Buf.resize(Passphrase.Length() * sizeof(wchar_t) + sizeof(wchar_t));
+    for (int Index = 0; Index <= Passphrase.Length(); Index++)
+    {
+      Buf[(Index * 2)] = (Passphrase.c_str()[Index] >> 8);
+      Buf[(Index * 2) + 1] = (Passphrase.c_str()[Index] & 0x00FF);
+    }
+
     bool Result =
-      (PKCS12_parse(Pkcs12, UTF8String(Passphrase).c_str(), &PrivateKey, &Certificate, NULL) == 1);
+      (PKCS12_parse(Pkcs12, &Buf[0], &PrivateKey, &Certificate, NULL) == 1);
     PKCS12_free(Pkcs12);
 
     if (!Result)