513166 - Simple Paged result doesn't provide the server's estimate

of the total number of entries

https://bugzilla.redhat.com/show_bug.cgi?id=513166

Description: By definition, the first integer of the response
control is for the estimated search result set size.  This fix
sets the estimated result set size.
 * controlType = LDAP_CONTROL_PAGEDRESULTS;
 * criticality = n/a;
 * controlValue:
 *   realSearchControlValue ::= SEQUENCE {
 *   size INTEGER (0..maxInt),
 *   -- result set size estimate from server
 *   cookie OCTET STRING

ldap/servers/slapd/back-ldbm/ldbm_search.c

@@ -108,6 +108,8 @@ berval_done(struct berval *val)
  */
 int ldbm_back_search_cleanup(Slapi_PBlock *pb, struct ldbminfo *li, sort_spec_thing *sort_control, int ldap_result, char* ldap_result_description, int function_result, Slapi_DN *sdn, struct vlv_request *vlv_request_control)
 {
+    int estimate = 0; /* estimated search result count */
+
     if(sort_control!=NULL)
     {
         sort_spec_free(sort_control);
@@ -125,6 +127,7 @@ int ldbm_back_search_cleanup(Slapi_PBlock *pb, struct ldbminfo *li, sort_spec_th
             /* in case paged results, clean up the conn */
             pagedresults_set_search_result(pb->pb_conn, NULL);
             slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, NULL );
+            slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate );
             delete_search_result_set(&sr);
         }
     }
@@ -160,6 +163,7 @@ ldbm_back_search( Slapi_PBlock *pb )
     LDAPControl **controls = NULL;
     Slapi_Operation *operation;
     entry_address *addr;
+    int estimate = 0; /* estimated search result set size */
 
     /* SORT control stuff */
     int sort = 0;
@@ -202,6 +206,7 @@ ldbm_back_search( Slapi_PBlock *pb )
     sr = new_search_result_set( NULL, 0, 
                                 compute_lookthrough_limit( pb, li ));
     slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, sr );
+    slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate );
     
     /* clear this out so we can free it later */
     memset(&vlv_request_control, 0, sizeof(vlv_request_control));
@@ -675,6 +680,12 @@ ldbm_back_search( Slapi_PBlock *pb )
     sr->sr_candidates = candidates;
     sr->sr_virtuallistview = virtual_list_view;
 
+    /* Set the estimated search result count for simple paged results */
+    if (sr->sr_candidates && !ALLIDS(sr->sr_candidates)) {
+        estimate = IDL_NIDS(sr->sr_candidates);
+    }
+    slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate );
+
     /* check to see if we can skip the filter test */
     if ( li->li_filter_bypass && NULL != candidates && !virtual_list_view
                 && !lookup_returned_allids ) {
@@ -1117,6 +1128,7 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
     Slapi_DN               basesdn;
     char                   *target_uniqueid;
     int                    rc = 0; 
+    int                    estimate = 0; /* estimated search result count */
 
     slapi_pblock_get( pb, SLAPI_BACKEND, &be );
     slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li );
@@ -1178,6 +1190,7 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
             /* in case paged results, clean up the conn */
             pagedresults_set_search_result(pb->pb_conn, NULL);
             slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, NULL );
+            slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate );
             if ( use_extension ) {
                 slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, NULL );
             }
@@ -1195,6 +1208,7 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
             /* in case paged results, clean up the conn */
             pagedresults_set_search_result(pb->pb_conn, NULL);
             slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, NULL );
+            slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate );
             if ( use_extension ) {
                 slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, NULL );
             } 
@@ -1211,6 +1225,7 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
             /* in case paged results, clean up the conn */
             pagedresults_set_search_result(pb->pb_conn, NULL);
             slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, NULL );
+            slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate );
             if ( use_extension ) {
                 slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, NULL );
             } 
@@ -1229,6 +1244,7 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
             /* in case paged results, clean up the conn */
             pagedresults_set_search_result(pb->pb_conn, NULL);
             slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, NULL );
+            slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate );
             if ( use_extension ) {
                 slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, NULL );
             }
@@ -1371,6 +1387,7 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
                          /* in case paged results, clean up the conn */
                          pagedresults_set_search_result(pb->pb_conn, NULL);
                          slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, NULL );
+                         slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate );
                          delete_search_result_set( &sr );
                          slapi_send_ldap_result( pb, LDAP_SIZELIMIT_EXCEEDED, NULL, NULL, nentries, urls );
                          rc = SLAPI_FAIL_GENERAL;

ldap/servers/slapd/dse.c

@@ -1660,6 +1660,7 @@ dse_search(Slapi_PBlock *pb) /* JCM There should only be one exit point from thi
     int isrootdse= 0;
     char returntext[SLAPI_DSE_RETURNTEXT_SIZE]= "";
 	Slapi_DN basesdn;
+	int estimate = 0; /* estimated search result set size */
 
     /* 
      * Get private information created in the init routine. 
@@ -1716,6 +1717,7 @@ dse_search(Slapi_PBlock *pb) /* JCM There should only be one exit point from thi
 				slapi_pblock_set (pb, SLAPI_SEARCH_RESULT_SET, ss);
 				dse_search_set_add_entry (ss, baseentry); /* consumes the entry */
 				baseentry= NULL;
+				estimate = 1; /* scope base */
         	}
         }
 		slapi_entry_free(baseentry);
@@ -1726,10 +1728,11 @@ dse_search(Slapi_PBlock *pb) /* JCM There should only be one exit point from thi
     case LDAP_SCOPE_SUBTREE:
         if(!isrootdse)
         {
-            do_dse_search(pdse, pb, scope, &basesdn, filter, attrs, attrsonly);
+            estimate = do_dse_search(pdse, pb, scope, &basesdn, filter, attrs, attrsonly);
         }
         break;
     }
+	slapi_pblock_set (pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate);
 
     /* Search is done, send LDAP_SUCCESS */
 	slapi_sdn_done(&basesdn);

ldap/servers/slapd/opshared.c

@@ -217,6 +217,7 @@ op_shared_search (Slapi_PBlock *pb, int send_result)
   unsigned int    pr_stat = 0;
 
   ber_int_t pagesize = -1;
+  ber_int_t estimate = 0; /* estimated search result set size */
   int curr_search_count = 0;
   Slapi_Backend *pr_be = NULL;
   void *pr_search_result = NULL;
@@ -376,6 +377,8 @@ op_shared_search (Slapi_PBlock *pb, int send_result)
               pr_search_result = pagedresults_get_search_result(pb->pb_conn);
               pr_search_result_count =
                              pagedresults_get_search_result_count(pb->pb_conn);
+              estimate = 
+                 pagedresults_get_search_result_set_size_estimate(pb->pb_conn);
           } else {
               /* parse paged-results-control failed */
               if (iscritical) { /* return an error since it's critical */
@@ -572,13 +575,16 @@ op_shared_search (Slapi_PBlock *pb, int send_result)
         } else {
           curr_search_count = pnentries;
         }
+        estimate = 0;
       } else {
         curr_search_count = pnentries;
+        estimate -= estimate?curr_search_count:0;
       }
-      pagedresults_set_response_control(pb, 0, pagesize, curr_search_count);
+      pagedresults_set_response_control(pb, 0, estimate, curr_search_count);
       if (pagedresults_get_with_sort(pb->pb_conn)) {
         sort_make_sort_response_control(pb, CONN_GET_SORT_RESULT_CODE, NULL);
       }
+      pagedresults_set_search_result_set_size_estimate(pb->pb_conn, estimate);
       next_be = NULL; /* to break the loop */
     } else {
       /* be_suffix null means that we are searching the default backend
@@ -706,16 +712,19 @@ op_shared_search (Slapi_PBlock *pb, int send_result)
             } else {
               curr_search_count = pnentries;
               slapi_pblock_get(pb, SLAPI_SEARCH_RESULT_SET, &sr);
+              slapi_pblock_get(pb, SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE, &estimate);
               if (pagedresults_set_current_be(pb->pb_conn, be) < 0 ||
                   pagedresults_set_search_result(pb->pb_conn, sr) < 0 ||
                   pagedresults_set_search_result_count(pb->pb_conn,
                                                    curr_search_count) < 0 ||
+                  pagedresults_set_search_result_set_size_estimate(pb->pb_conn,
+                                                   estimate) < 0 ||
                   pagedresults_set_with_sort(pb->pb_conn, with_sort) < 0) {
                 goto free_and_return;
               }
             }
             pagedresults_set_response_control(pb, 0,
-                                              pagesize, curr_search_count);
+                                              estimate, curr_search_count);
             slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, NULL );
             next_be = NULL; /* to break the loop */
         }

ldap/servers/slapd/pagedresults.c

@@ -114,7 +114,7 @@ pagedresults_parse_control_value( struct berval *psbvp,
  */
 void
 pagedresults_set_response_control( Slapi_PBlock *pb, int iscritical,
-                                    ber_int_t pagesize, int curr_search_count )
+                                   ber_int_t estimate, int curr_search_count )
 {
     LDAPControl **resultctrls = NULL;
     LDAPControl pr_respctrl;
@@ -135,7 +135,7 @@ pagedresults_set_response_control( Slapi_PBlock *pb, int iscritical,
     } else {
         cookie_str = slapi_ch_smprintf("%d", curr_search_count);
     }
-    ber_printf ( ber, "{io}", pagesize, cookie_str, strlen(cookie_str) );
+    ber_printf ( ber, "{io}", estimate, cookie_str, strlen(cookie_str) );
     if ( ber_flatten ( ber, &berval ) != LDAP_SUCCESS )
     {
         goto bailout;
@@ -249,6 +249,31 @@ pagedresults_set_search_result_count(Connection *conn, int count)
     return rc;
 }
 
+int
+pagedresults_get_search_result_set_size_estimate(Connection *conn)
+{
+    int count = 0;
+    if (conn) {
+        PR_Lock(conn->c_mutex);
+        count = conn->c_search_result_set_size_estimate;
+        PR_Unlock(conn->c_mutex);
+    }
+    return count;
+}
+
+int
+pagedresults_set_search_result_set_size_estimate(Connection *conn, int count)
+{
+    int rc = -1;
+    if (conn) {
+        PR_Lock(conn->c_mutex);
+        conn->c_search_result_set_size_estimate = count;
+        PR_Unlock(conn->c_mutex);
+        rc = 0;
+    }
+    return rc;
+}
+
 int
 pagedresults_get_with_sort(Connection *conn)
 {

ldap/servers/slapd/pblock.c

@@ -1522,6 +1522,13 @@ slapi_pblock_get( Slapi_PBlock *pblock, int arg, void *value )
 			(*(void **)value) = pblock->pb_op->o_results.r.r_search.search_result_set;
 		}
 		break;
+	/* estimated search result set size */
+	case SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE:
+		if(pblock->pb_op!=NULL)
+		{
+			(*(int *)value) = pblock->pb_op->o_results.r.r_search.estimate;
+		}
+		break;
 	/* Entry returned from iterating over results set */
 	case SLAPI_SEARCH_RESULT_ENTRY:
 		if(pblock->pb_op!=NULL)
@@ -2891,6 +2898,13 @@ slapi_pblock_set( Slapi_PBlock *pblock, int arg, void *value )
 			pblock->pb_op->o_results.r.r_search.search_result_set = (void *)value;
 		}
 		break;
+	/* estimated search result set size */
+	case SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE:
+		if(pblock->pb_op!=NULL)
+		{
+			pblock->pb_op->o_results.r.r_search.estimate = *(int *)value;
+		}
+		break;
 	/* Search result - entry returned from iterating over result set */
 	case SLAPI_SEARCH_RESULT_ENTRY:
 		if(pblock->pb_op!=NULL)

ldap/servers/slapd/proto-slap.h

@@ -1358,13 +1358,15 @@ int slapd_do_all_nss_ssl_init(int slapd_exemode, int importexport_encrypt,
  * pagedresults.c
  */
 int pagedresults_parse_control_value(struct berval *psbvp, ber_int_t *pagesize, int *curr_search_count);
-void pagedresults_set_response_control(Slapi_PBlock *pb, int	iscritical, ber_int_t pagesize, int curr_search_count);
+void pagedresults_set_response_control(Slapi_PBlock *pb, int iscritical, ber_int_t estimate, int curr_search_count);
 Slapi_Backend *pagedresults_get_current_be(Connection *conn);
 int pagedresults_set_current_be(Connection *conn, Slapi_Backend *be);
 void *pagedresults_get_search_result(Connection *conn);
 int pagedresults_set_search_result(Connection *conn, void *sr);
 int pagedresults_get_search_result_count(Connection *conn);
 int pagedresults_set_search_result_count(Connection *conn, int cnt);
+int pagedresults_get_search_result_get_size_estimate(Connection *conn);
+int pagedresults_set_search_result_set_size_estimate(Connection *conn, int cnt);
 int pagedresults_get_with_sort(Connection *conn);
 int pagedresults_set_with_sort(Connection *conn, int flags);
 int pagedresults_get_sort_result_code(Connection *conn);

ldap/servers/slapd/slap.h

@@ -1230,6 +1230,8 @@ typedef struct slapi_operation_results
 			int nentries;
 			/* Any referrals encountered during the search */
 			struct berval **search_referrals;
+			/* estimated search result set size */
+			int estimate;
 		} r_search;
 
 		struct extended_results
@@ -1345,6 +1347,7 @@ typedef struct conn {
     Slapi_Backend   *c_current_be;         /* backend being used */
     void            *c_search_result_set;  /* search result set for paging */
     int             c_search_result_count; /* search result count */
+    int             c_search_result_set_size_estimate; /* estimated search result set size */
     int             c_sort_result_code;    /* sort result put in response */
     time_t          c_timelimit;           /* time limit for this connection */
     /* PAGED_RESULTS ENDS */

ldap/servers/slapd/slapi-plugin.h

@@ -5841,6 +5841,8 @@ typedef struct slapi_plugindesc {
  */
 /* Search result set */
 #define SLAPI_SEARCH_RESULT_SET			193
+/* Estimated search result set size (for paged results) */
+#define SLAPI_SEARCH_RESULT_SET_SIZE_ESTIMATE	1930
 /* Search result - next entry returned from search result set */
 #define	SLAPI_SEARCH_RESULT_ENTRY		194
 #define SLAPI_SEARCH_RESULT_ENTRY_EXT           1944