Workaround bogus base64 encoded passwords that end in newline

https://bugzilla.redhat.com/show_bug.cgi?id=552421
Resolves: bug 552421
Bug Description: Cannot log into admin server after upgrade (fedora-ds-admin-1.1.6 -> 389-admin-1.1.9
Reviewed by: nkinder (Thanks!)
Branch: HEAD
Fix Description: Some older versions of setup encoded the admin password in SHA and added a trailing newline to the userPassword attribute when adding the admin entry.  This changes the SHA passsword compare routine to ignore a trailing
newline character in the dbpwd.  newline is not a valid base64 character.
Platforms tested: RHEL5 x86_64
Flag Day: no
Doc impact: no

ldap/servers/plugins/pwdstorage/pwd_util.c

@@ -50,10 +50,14 @@
  * calculate the number of bytes the base64 encoded encval
  * will have when decoded, taking into account padding
  */
-int
-pwdstorage_base64_decode_len(const char *encval)
+PRUint32
+pwdstorage_base64_decode_len(const char *encval, PRUint32 enclen)
 {
-    int len = strlen(encval);
+    PRUint32 len = enclen;
+
+    if (len == 0) {
+        len = strlen(encval);
+    }
     if (len && (0 == (len & 3))) {
         if('=' == encval[len - 1]) {
             if('=' == encval[len - 2]) {

ldap/servers/plugins/pwdstorage/pwdstorage.h

@@ -113,6 +113,6 @@ int smd5_pw_cmp( const char *userpwd, const char *dbpwd );
 char *smd5_pw_enc( const char *pwd );
 
 /* Utility functions */
-int pwdstorage_base64_decode_len(const char *encval);
+PRUint32 pwdstorage_base64_decode_len(const char *encval, PRUint32 enclen);
 
 #endif /* _PWDSTORAGE_H */

ldap/servers/plugins/pwdstorage/sha_pwd.c

@@ -83,6 +83,7 @@ sha_pw_cmp (const char *userpwd, const char *dbpwd, unsigned int shaLen )
     unsigned int secOID;
     char *schemeName;
     char *hashresult = NULL;
+    PRUint32 dbpwd_len;
 
     /* Determine which algorithm we're using */
     switch (shaLen) {
@@ -107,17 +108,25 @@ sha_pw_cmp (const char *userpwd, const char *dbpwd, unsigned int shaLen )
             goto loser;
     }
 
+    /* in some cases, the password was stored incorrectly - the base64 dbpwd ends
+       in a newline - we check for this case and remove the newline, if any -
+       see bug 552421 */
+    dbpwd_len = strlen(dbpwd);
+    if ((dbpwd_len > 0) && (dbpwd[dbpwd_len-1] == '\n')) {
+        dbpwd_len--;
+    }
+
     /*
      * Decode hash stored in database.
      */
-    hash_len = pwdstorage_base64_decode_len(dbpwd);
+    hash_len = pwdstorage_base64_decode_len(dbpwd, dbpwd_len);
     if ( hash_len > sizeof(quick_dbhash) ) { /* get more space: */
         dbhash = (char*) slapi_ch_calloc( hash_len, sizeof(char) );
         if ( dbhash == NULL ) goto loser;
     } else {
         memset( quick_dbhash, 0, sizeof(quick_dbhash) );
     }
-    hashresult = PL_Base64Decode( dbpwd, 0, dbhash );
+    hashresult = PL_Base64Decode( dbpwd, dbpwd_len, dbhash );
     if (NULL == hashresult) {
         slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, hasherrmsg, schemeName, dbpwd );
         goto loser;

ldap/servers/plugins/pwdstorage/smd5_pwd.c

@@ -82,7 +82,7 @@ smd5_pw_cmp( const char *userpwd, const char *dbpwd )
    /*
     * Decode hash stored in database.
     */
-   hash_len = pwdstorage_base64_decode_len(dbpwd);
+   hash_len = pwdstorage_base64_decode_len(dbpwd, 0);
    if ( hash_len >= sizeof(quick_dbhash) ) { /* get more space: */
       dbhash = (char*) slapi_ch_calloc( hash_len + 1, sizeof(char) );
       if ( dbhash == NULL ) goto loser;