Ticket 47382 - Add a warning message when a connection hits the max number of threads

Bug Description:  Currently there is no way to tell when a connection is at max threads.

Fix Description:  Update cn=monitor, "cn=snmp,cn=monitor", snmp, etc to report on
                  the number of current active threads that are in a max thread state.
                  Report the number of times any connection has hit max threads.  For
                  cn=monitor, extend the "connection" attribute to include three more
                  elements:

                         "1:2:3"

			 1 = Connection max threads state:  1 is in max threads, 0 is not
			 2 = The number of times this thread has hit max threads
			 3 = The number of operations attempted that were blocked
			     by max threads.

                         connection: 66:20130724195333Z:1:1:-:cn=directory manager:0:1:3

https://fedorahosted.org/389/ticket/47382

Reviewed by: richm(Thanks!)

ldap/servers/slapd/agtmmap.c

@@ -456,6 +456,9 @@ agt_mread_stats (int hdl, struct hdr_stats_t *pHdrInfo, struct ops_stats_t *pDsO
         pDsOpsTbl->dsChainings             = pfile_stats->ops_stats.dsChainings;
         pDsOpsTbl->dsSecurityErrors        = pfile_stats->ops_stats.dsSecurityErrors;
         pDsOpsTbl->dsErrors                = pfile_stats->ops_stats.dsErrors;
+        pDsOpsTbl->dsConnections           = pfile_stats->ops_stats.dsConnections;
+        pDsOpsTbl->dsConnectionsInMaxThreads = pfile_stats->ops_stats.dsConnectionsInMaxThreads;
+        pDsOpsTbl->dsMaxThreadsHit         = pfile_stats->ops_stats.dsMaxThreadsHit;
     }
 
     if (pDsEntTbl != NULL) {

ldap/servers/slapd/agtmmap.h

@@ -145,6 +145,8 @@ struct ops_stats_t{
     PRUint64 dsErrors;
     PRUint64 dsConnections;	 /* Number of currently connected clients */
     PRUint64 dsConnectionSeq; /* Monotonically increasing number bumped on each new conn est */
+    PRUint64 dsMaxThreadsHit; /* Number of times a connection hit max threads */
+    PRUint64 dsConnectionsInMaxThreads; /* current number of connections that are in max threads */
     PRUint64 dsBytesRecv;	/* Count of bytes read from clients */
     PRUint64 dsBytesSent;	/* Count of bytes sent to clients */
     PRUint64 dsEntriesReturned; /* Number of entries returned by the server */

ldap/servers/slapd/connection.c

@@ -2285,8 +2285,8 @@ connection_threadmain()
 	int more_data = 0;
 	int replication_connection = 0; /* If this connection is from a replication supplier, we want to ensure that operation processing is serialized */
 	int doshutdown = 0;
+	int maxthreads = 0;
 	long bypasspollcnt = 0;
-	long maxconnhits = 0;
 
 #if defined( OSF1 ) || defined( hpux )
 	/* Arrange to ignore SIGPIPE signals. */
@@ -2354,7 +2354,7 @@ connection_threadmain()
 		/* Once we're here we have a pb */ 
 		conn = pb->pb_conn;
 		op = pb->pb_op;
-		
+		maxthreads = config_get_maxthreadsperconn();
 		more_data = 0;
 		ret = connection_read_operation(conn,op,&tag,&more_data);
 
@@ -2426,6 +2426,7 @@ connection_threadmain()
 		replication_connection = conn->c_isreplication_session;
 		if ((tag != LDAP_REQ_UNBIND) && !thread_turbo_flag && !replication_connection) {
 			if (!more_data) {
+				conn->c_flags &= ~CONN_FLAG_MAX_THREADS;
 				connection_make_readable_nolock(conn);
 				/* once the connection is readable, another thread may access conn,
 				 * so need locking from here on */
@@ -2434,16 +2435,17 @@ connection_threadmain()
 				bypasspollcnt++;
 				PR_Lock(conn->c_mutex);
 				/* don't do this if it would put us over the max threads per conn */
-				if (conn->c_threadnumber < config_get_maxthreadsperconn()) {
+				if (conn->c_threadnumber < maxthreads) {
 					/* for turbo, c_idlesince is set above - for !turbo and
 					 * !more_data, we put the conn back in the poll loop and
 					 * c_idlesince is set in handle_pr_read_ready - since we
 					 * are bypassing both of those, we set idlesince here
 					 */
 					conn->c_idlesince = curtime;
-					connection_activity(conn);
+					connection_activity(conn, maxthreads);
 				} else {
-					maxconnhits++;
+					/* keep count of how many times maxthreads has blocked an operation */
+					conn->c_maxthreadsblocked++;
 				}
 				PR_Unlock(conn->c_mutex);
 			}
@@ -2485,6 +2487,8 @@ done:
 			connection_remove_operation_ext(pb, conn, op);
 			connection_make_readable_nolock(conn);
 			conn->c_threadnumber--;
+			slapi_counter_decrement(conns_in_maxthreads);
+			slapi_counter_decrement(g_get_global_snmp_vars()->ops_tbl.dsConnectionsInMaxThreads);
 			connection_release_nolock(conn);
 			PR_Unlock(conn->c_mutex);
 			signal_listner();
@@ -2528,11 +2532,17 @@ done:
 						need_wakeup = 1;
 					}
 					if (!need_wakeup) {
-						if (conn->c_threadnumber == config_get_maxthreadsperconn())
+						if (conn->c_threadnumber == maxthreads)
 							need_wakeup = 1;
 						else
 							need_wakeup = 0;
 					}
+
+					if(conn->c_threadnumber == maxthreads){
+					    conn->c_flags &= ~CONN_FLAG_MAX_THREADS;
+					    slapi_counter_decrement(conns_in_maxthreads);
+					    slapi_counter_decrement(g_get_global_snmp_vars()->ops_tbl.dsConnectionsInMaxThreads);
+					}
 					conn->c_threadnumber--;
 					connection_release_nolock(conn);
 					/* Call signal_listner after releasing the
@@ -2552,7 +2562,7 @@ done:
 
 /* thread need to hold conn->c_mutex before calling this function */
 int
-connection_activity(Connection *conn)
+connection_activity(Connection *conn, int maxthreads)
 {
 	struct Slapi_op_stack *op_stack_obj;
 
@@ -2568,6 +2578,14 @@ connection_activity(Connection *conn)
 	/* set these here so setup_pr_read_pds will not add this conn back to the poll array */
 	conn->c_gettingber = 1;
 	conn->c_threadnumber++;
+	if(conn->c_threadnumber == maxthreads){
+		conn->c_flags |= CONN_FLAG_MAX_THREADS;
+		conn->c_maxthreadscount++;
+		slapi_counter_increment(max_threads_count);
+		slapi_counter_increment(conns_in_maxthreads);
+		slapi_counter_increment(g_get_global_snmp_vars()->ops_tbl.dsConnectionsInMaxThreads);
+		slapi_counter_increment(g_get_global_snmp_vars()->ops_tbl.dsMaxThreadsHit);
+	}
 	op_stack_obj = connection_get_operation();
 	connection_add_operation(conn, op_stack_obj->op);
 	/* Add conn to the end of the work queue.  */

ldap/servers/slapd/conntable.c

@@ -376,6 +376,7 @@ void
 connection_table_as_entry(Connection_Table *ct, Slapi_Entry *e)
 {
 	char buf[BUFSIZ];
+	char maxthreadbuf[BUFSIZ];
 	struct berval val;
 	struct berval	*vals[2];
 	int	i, nconns, nreadwaiters;
@@ -405,6 +406,11 @@ connection_table_as_entry(Connection_Table *ct, Slapi_Entry *e)
 			int lendn = ct->c[i].c_dn ? strlen(ct->c[i].c_dn) : 6; /* "NULLDN" */
 			char *bufptr = &buf[0];
 			char *newbuf = NULL;
+			int maxthreadstate = 0;
+
+			if(ct->c[i].c_flags & CONN_FLAG_MAX_THREADS){
+				maxthreadstate = 1;
+			}
 
 			nconns++;
 			if ( ct->c[i].c_gettingber )
@@ -423,23 +429,38 @@ connection_table_as_entry(Connection_Table *ct, Slapi_Entry *e)
 #endif
 			strftime( buf2, sizeof(buf2), "%Y%m%d%H%M%SZ", &utm );
 
-			if (lendn > (BUFSIZ - 46)) { 
+			/*
+			 * Max threads per connection stats
+			 *
+			 * Appended output "1:2:3"
+			 *
+			 * 1 = Connection max threads state:  1 is in max threads, 0 is not
+			 * 2 = The number of times this thread has hit max threads
+			 * 3 = The number of operations attempted that were blocked
+			 *     by max threads.
+			 */
+			PR_snprintf(maxthreadbuf, sizeof(maxthreadbuf), "%d:%"NSPRIu64":%"NSPRIu64"",
+				maxthreadstate, ct->c[i].c_maxthreadscount, ct->c[i].c_maxthreadsblocked);
+
+			if ((lendn + strlen(maxthreadbuf)) > (BUFSIZ - 46)) {
 				/*
 				 * 46 = 4 for the "i" couter + 20 for buf2 +
 				 *     10 for c_opsinitiated + 10 for c_opscompleted +
 				 *      1 for c_gettingber + 1
 				 */
-				newbuf = (char *) slapi_ch_malloc(lendn + 46);
+				newbuf = (char *) slapi_ch_malloc(lendn + strlen(maxthreadbuf) + 46);
 				bufptr = newbuf;
 			}
 
-			sprintf( bufptr, "%d:%s:%d:%d:%s%s:%s", i,
+			sprintf( bufptr, "%d:%s:%d:%d:%s%s:%s:%s", i,
 			    buf2, 
 			    ct->c[i].c_opsinitiated, 
 			    ct->c[i].c_opscompleted,
-			    ct->c[i].c_gettingber ? "r" : "",
+			    ct->c[i].c_gettingber ? "r" : "-",
 			    "",
-			    ct->c[i].c_dn ? ct->c[i].c_dn : "NULLDN" );
+			    ct->c[i].c_dn ? ct->c[i].c_dn : "NULLDN",
+			    maxthreadbuf
+			    );
 			val.bv_val = bufptr;
 			val.bv_len = strlen( bufptr );
 			attrlist_merge( &e->e_attrs, "connection", vals );
@@ -458,6 +479,16 @@ connection_table_as_entry(Connection_Table *ct, Slapi_Entry *e)
 	val.bv_len = strlen( buf );
 	attrlist_replace( &e->e_attrs, "totalconnections", vals );
 
+	PR_snprintf( buf, sizeof(buf), "%" NSPRIu64, slapi_counter_get_value(conns_in_maxthreads));
+	val.bv_val = buf;
+	val.bv_len = strlen( buf );
+	attrlist_replace( &e->e_attrs, "currentconnectionsatmaxthreads", vals );
+
+	PR_snprintf( buf, sizeof(buf), "%" NSPRIu64, slapi_counter_get_value(max_threads_count));
+	val.bv_val = buf;
+	val.bv_len = strlen( buf );
+	attrlist_replace( &e->e_attrs, "maxthreadsperconnhits", vals );
+
 	PR_snprintf( buf, sizeof(buf), "%d", (ct!=NULL?ct->size:0) );
 	val.bv_val = buf;
 	val.bv_len = strlen( buf );

ldap/servers/slapd/daemon.c

@@ -1723,6 +1723,9 @@ setup_pr_read_pds(Connection_Table *ct, PRFileDesc **n_tcps, PRFileDesc **s_tcps
 				}
 				else
 				{
+					if(c->c_threadnumber >= max_threads_per_conn){
+						c->c_maxthreadsblocked++;
+					}
 					c->c_fdi = SLAPD_INVALID_SOCKET_INDEX;
 				}
 			}
@@ -1840,6 +1843,7 @@ handle_read_ready(Connection_Table *ct, fd_set *readfds)
 	time_t curtime = current_time();
 	slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
 	int idletimeout;
+	int maxthreads = config_get_maxthreadsperconn();
 
 #ifdef LDAP_DEBUG
 	if ( slapd_ldap_debug & LDAP_DEBUG_CONNS )
@@ -1870,7 +1874,7 @@ handle_read_ready(Connection_Table *ct, fd_set *readfds)
 					c->c_idlesince = curtime;
 
 					/* This is where the work happens ! */
-					connection_activity( c );
+					connection_activity( c, maxthreads);
 
 					/* idle timeout */
 				}
@@ -1898,6 +1902,7 @@ handle_pr_read_ready(Connection_Table *ct, PRIntn num_poll)
 	time_t curtime = current_time();
 	slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
 	int idletimeout;
+	int maxthreads = config_get_maxthreadsperconn();
 #if defined( XP_WIN32 )
 	int i;
 #endif
@@ -2021,7 +2026,7 @@ handle_pr_read_ready(Connection_Table *ct, PRIntn num_poll)
 
 					/* This is where the work happens ! */
 					/* MAB: 25 jan 01, error handling added */
-					if ((connection_activity( c )) == -1) {
+					if ((connection_activity( c, maxthreads )) == -1) {
 						/* This might happen as a result of
 						 * trying to acquire a closing connection
 						 */

ldap/servers/slapd/fe.h

@@ -58,6 +58,8 @@ extern __declspec(dllimport) int slapd_ldap_debug;
 #endif
 extern Slapi_Counter *ops_initiated;
 extern Slapi_Counter *ops_completed;
+extern Slapi_Counter *max_threads_count;
+extern Slapi_Counter *conns_in_maxthreads;
 extern PRThread *listener_tid;
 extern PRThread *listener_tid;
 extern Slapi_Counter *num_conns;
@@ -107,7 +109,7 @@ void SVRCORE_DestroyNTUserPinObj(SVRCORENTUserPinObj *obj);
  * connection.c
  */
 void connection_abandon_operations( Connection *conn );
-int connection_activity( Connection *conn );
+int connection_activity( Connection *conn, int maxthreads );
 void init_op_threads();
 int connection_new_private(Connection *conn);
 void connection_remove_operation( Connection *conn, Operation *op );

ldap/servers/slapd/globals.c

@@ -90,7 +90,8 @@ Slapi_PBlock	*repl_pb = NULL;
 Slapi_Counter	*ops_initiated;
 Slapi_Counter	*ops_completed;
 Slapi_Counter	*num_conns;
-
+Slapi_Counter	*max_threads_count;
+Slapi_Counter	*conns_in_maxthreads;
 
 /*
   DEC/COMPAQ has released a patch for 4.0d (e?) which will speed up
@@ -173,11 +174,15 @@ set_entry_points()
 	if (config_get_slapi_counters()) {
 		ops_initiated = slapi_counter_new();
 		ops_completed = slapi_counter_new();
+		max_threads_count = slapi_counter_new();
+		conns_in_maxthreads = slapi_counter_new();
 		g_set_num_entries_sent( slapi_counter_new() );
 		g_set_num_bytes_sent( slapi_counter_new() );
 	} else {
 		ops_initiated = NULL;
 		ops_completed = NULL;
+		max_threads_count = NULL;
+		conns_in_maxthreads = NULL;
 		g_set_num_entries_sent( NULL );
 		g_set_num_bytes_sent( NULL );
 	}

ldap/servers/slapd/ntperfdll/nsldapctr.cpp

@@ -260,6 +260,30 @@ NS_DATA_DEFINITION NSDataDefinition = {
 		CONNECTIONS_OFFSET
     },
     {   
+		sizeof(PERF_COUNTER_DEFINITION),
+		CONNECTIONSMAXTHREADS,
+		0,
+		CONNECTIONSMAXTHREADS,
+		0,
+		0,
+		PERF_DETAIL_NOVICE,
+		PERF_COUNTER_RAWCOUNT,
+		sizeof(DWORD),
+		CONNECTIONSMAXTHREADS_OFFSET
+    },
+    {
+		sizeof(PERF_COUNTER_DEFINITION),
+		CONNECTIONSHITMAXTHREADS,
+		0,
+		CONNECTIONSHITMAXTHREADS,
+		0,
+		0,
+		PERF_DETAIL_NOVICE,
+		PERF_COUNTER_RAWCOUNT,
+		sizeof(DWORD),
+		CONNECTIONSHITMAXTHREADS_OFFSET
+    },
+    {
     	sizeof(PERF_COUNTER_DEFINITION),
 		BIND_RATE,
 	    0,
@@ -755,6 +779,10 @@ OpenNSPerformanceData(LPWSTR lpDeviceNames)
         NSDataDefinition.moddn_rate.CounterHelpTitleIndex += dwFirstHelp;
         NSDataDefinition.connections.CounterNameTitleIndex += dwFirstCounter;
         NSDataDefinition.connections.CounterHelpTitleIndex += dwFirstHelp;
+        NSDataDefinition.connectionsmaxthreads.CounterNameTitleIndex += dwFirstCounter;
+        NSDataDefinition.connectionsmaxthreads.CounterHelpTitleIndex += dwFirstHelp;
+        NSDataDefinition.connectionshitmaxthreads.CounterNameTitleIndex += dwFirstCounter;
+        NSDataDefinition.connectionshitmaxthreads.CounterHelpTitleIndex += dwFirstHelp;
         NSDataDefinition.bind_rate.CounterNameTitleIndex += dwFirstCounter;
         NSDataDefinition.bind_rate.CounterHelpTitleIndex += dwFirstHelp;
         NSDataDefinition.entries_returned.CounterNameTitleIndex += dwFirstCounter;
@@ -829,6 +857,8 @@ struct _status_struct_s {
 	DWORD	compare_rate;
 	DWORD	moddn_rate;
 	DWORD	connections;
+	DWORD	connectionsmaxthreads;
+	DWORD	connectionshitmaxthreads;
 	DWORD	bind_rate;
 	DWORD	entries_returned;
 	DWORD	entries_returned_rate;
@@ -857,6 +887,8 @@ Get_Actual_Data(agt_stats_t *smem,
 	results->connections = 0;
 	results->tot_errs = pOpsStats->dsErrors ;
 	results->connections = pOpsStats->dsConnections ;
+	results->connectionsmaxthreads = pOpsStats->dsConnectionsInMaxThreads ;
+	results->connectionshitmaxthreads = pOpsStats->dsMaxThreadsHit ;
 	results->tot_bytes_written = pOpsStats->dsBytesSent ;
 	results->tot_bytes_read = pOpsStats->dsBytesRecv ;
 	results->throughput = pOpsStats->dsBytesSent +  pOpsStats->dsBytesRecv;

ldap/servers/slapd/ntperfdll/nsldapctrdef.h

@@ -67,4 +67,6 @@
 #define REFERRALS_RETURNED_RATE	36
 #define BYTES_READ_RATE 38
 #define BYTES_WRITTEN_RATE 40
+#define CONNECTIONSMAXTHREADS 42
+#define CONNECTIONSHITMAXTHREADS 44
 

ldap/servers/slapd/ntperfdll/nsldapctrs.h

@@ -73,11 +73,13 @@
 #define BYTES_READ_RATE_OFFSET		REFERRALS_RETURNED_RATE_OFFSET + sizeof(DWORD)
 #define BYTES_WRITTEN_RATE_OFFSET	BYTES_READ_RATE_OFFSET + sizeof(DWORD)
 #define SIZE_OF_NS_PERFORMANCE_DATA     	BYTES_WRITTEN_RATE_OFFSET + sizeof(DWORD)
+#define CONNECTIONSMAXTHREADS_OFFSET		SIZE_OF_NS_PERFORMANCE_DATA + sizeof(DWORD)
+#define CONNECTIONSHITMAXTHREADS_OFFSET	CONNECTIONSMAXTHREADS_OFFSET + sizeof(DWORD)
 
 typedef struct _NS_DATA_DEFINITION {
-    	PERF_OBJECT_TYPE	NS_ObjectType;
-    	PERF_COUNTER_DEFINITION	connection_rate;
-    	PERF_COUNTER_DEFINITION	throughput;
+	PERF_OBJECT_TYPE	NS_ObjectType;
+	PERF_COUNTER_DEFINITION	connection_rate;
+	PERF_COUNTER_DEFINITION	throughput;
 	PERF_COUNTER_DEFINITION total_bytes_written;
 	PERF_COUNTER_DEFINITION total_bytes_read;
 	PERF_COUNTER_DEFINITION	operation_rate;
@@ -89,6 +91,8 @@ typedef struct _NS_DATA_DEFINITION {
 	PERF_COUNTER_DEFINITION compare_rate;
 	PERF_COUNTER_DEFINITION moddn_rate;
 	PERF_COUNTER_DEFINITION connections;
+	PERF_COUNTER_DEFINITION connectionsmaxthreads;
+	PERF_COUNTER_DEFINITION connectionshitmaxthreads;
 	PERF_COUNTER_DEFINITION bind_rate;
 	PERF_COUNTER_DEFINITION entries_returned;
 	PERF_COUNTER_DEFINITION entries_returned_rate;

ldap/servers/slapd/slap.h

@@ -1448,7 +1448,9 @@ typedef struct conn {
 	int				c_gettingber;	/* in the middle of ber_get_next  */
 	BerElement		*c_currentber;	/* ber we're getting              */
 	time_t			c_starttime;	/* when the connection was opened */
-	PRUint64	c_connid;	/* id of this connection for stats*/
+	PRUint64		c_connid;	/* id of this connection for stats*/
+	PRUint64		c_maxthreadscount; /* # of times a conn hit max threads */
+	PRUint64		c_maxthreadsblocked; /* # of operations blocked by maxthreads */
 	int				c_opsinitiated;	/* # ops initiated/next op id	  */
 	PRInt32			c_opscompleted;	/* # ops completed		  */
 	PRInt32			c_threadnumber; /* # threads used in this conn    */
@@ -1517,6 +1519,9 @@ typedef struct conn {
                                               * processing a pagedresults search
                                               */
 #define CONN_FLAG_PAGEDRESULTS_ABANDONED  512/* pagedresults abandoned */
+
+#define CONN_FLAG_MAX_THREADS 1024 /* Flag set when connection is at the maximum number of threads */
+
 #define CONN_GET_SORT_RESULT_CODE (-1)
 
 #define START_TLS_OID    "1.3.6.1.4.1.1466.20037"
@@ -1831,6 +1836,8 @@ struct snmp_ops_tbl_t{
     Slapi_Counter *dsBytesSent;	/* Count of bytes sent to clients */
     Slapi_Counter *dsEntriesReturned;
     Slapi_Counter *dsReferralsReturned;
+    Slapi_Counter *dsMaxThreadsHit;
+    Slapi_Counter *dsConnectionsInMaxThreads;
 };
 
 struct snmp_entries_tbl_t{

ldap/servers/slapd/snmp_collator.c

@@ -161,6 +161,8 @@ static int snmp_collator_init(){
 	g_get_global_snmp_vars()->ops_tbl.dsBytesSent			= slapi_counter_new();
 	g_get_global_snmp_vars()->ops_tbl.dsEntriesReturned		= slapi_counter_new();
 	g_get_global_snmp_vars()->ops_tbl.dsReferralsReturned		= slapi_counter_new();
+	g_get_global_snmp_vars()->ops_tbl.dsConnectionsInMaxThreads	= slapi_counter_new();
+	g_get_global_snmp_vars()->ops_tbl.dsMaxThreadsHit		= slapi_counter_new();
 	g_get_global_snmp_vars()->entries_tbl.dsMasterEntries		= slapi_counter_new();
 	g_get_global_snmp_vars()->entries_tbl.dsCopyEntries		= slapi_counter_new();
 	g_get_global_snmp_vars()->entries_tbl.dsCacheEntries		= slapi_counter_new();
@@ -672,6 +674,8 @@ snmp_update_ops_table()
     stats->ops_stats.dsErrors = slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsErrors);
     stats->ops_stats.dsConnections = slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsConnections);
     stats->ops_stats.dsConnectionSeq = slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsConnectionSeq);
+    stats->ops_stats.dsConnectionsInMaxThreads = slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsConnectionsInMaxThreads);
+    stats->ops_stats.dsMaxThreadsHit = slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsMaxThreadsHit);
     stats->ops_stats.dsBytesRecv = slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsBytesRecv);
     stats->ops_stats.dsBytesSent = slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsBytesSent);
     stats->ops_stats.dsEntriesReturned = slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsEntriesReturned);
@@ -818,6 +822,8 @@ snmp_as_entry(Slapi_Entry *e)
 	add_counter_to_value(e,"Errors", slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsErrors));
 	add_counter_to_value(e,"Connections", slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsConnections));
 	add_counter_to_value(e,"ConnectionSeq", slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsConnectionSeq));
+	add_counter_to_value(e,"ConnectionsInMaxThreads", slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsConnectionsInMaxThreads));
+	add_counter_to_value(e,"ConnectionsMaxThreadsCount", slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsMaxThreadsHit));
 	add_counter_to_value(e,"BytesRecv", slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsBytesRecv));
 	add_counter_to_value(e,"BytesSent", slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsBytesSent));
 	add_counter_to_value(e,"EntriesReturned", slapi_counter_get_value(g_get_global_snmp_vars()->ops_tbl.dsEntriesReturned));

ldap/servers/snmp/ldap-agent.c

@@ -513,6 +513,18 @@ dsOpsTable_get_value(netsnmp_request_info *request,
         the_stat = &context->ops_tbl.dsErrors;
         break;
 
+    case COLUMN_DSCONNECTIONS:
+        the_stat = &context->ops_tbl.dsConnections;
+        break;
+
+    case COLUMN_DSCONNECTIONSINMAXTHREADS:
+        the_stat = &context->ops_tbl.dsConnectionsInMaxThreads;
+        break;
+
+    case COLUMN_DSMAXTHREADSHIT:
+        the_stat = &context->ops_tbl.dsMaxThreadsHit;
+        break;
+
     default:/* We shouldn't get here */
         snmp_log(LOG_ERR, "Unknown column in dsOpsTable_get_value\n");
         return SNMP_ERR_GENERR;

ldap/servers/snmp/ldap-agent.h

@@ -186,8 +186,11 @@ typedef struct stats_table_context_s {
 #define COLUMN_DSCHAININGS 18
 #define COLUMN_DSSECURITYERRORS 19
 #define COLUMN_DSERRORS 20
+#define COLUMN_DSCONNECTIONS 21
+#define COLUMN_DSCONNECTIONSINMAXTHREADS 22
+#define COLUMN_DSMAXTHREADSHIT 23
 #define dsOpsTable_COL_MIN 1
-#define dsOpsTable_COL_MAX 20
+#define dsOpsTable_COL_MAX 23
 
 /*************************************************************
  * dsEntriesTable column defines

ldap/servers/snmp/ntagt/nsldapagt_nt.c

@@ -1069,6 +1069,8 @@ int MagtReadStats(MagtHdrInfo_t *hdrInfo,
     OpsTblInfo->Chainings             = pfile_stats->ops_stats.dsChainings;
     OpsTblInfo->SecurityErrors        = pfile_stats->ops_stats.dsSecurityErrors;
     OpsTblInfo->Errors                = pfile_stats->ops_stats.dsErrors;
+    OpsTblInfo->ConnectionsInMaxThreads = pfile_stats->ops_stats.dsConnectionsInMaxThreads;
+    OpsTblInfo->ConnectionsMaxThreadsHit = pfile_stats->ops_stats.dsMaxThreadsHit;
    }
   if(EntriesTblInfo != NULL){
     EntriesTblInfo->MasterEntries = pfile_stats->entries_stats.dsMasterEntries;

ldap/servers/snmp/ntagt/nsldapagt_nt.h

@@ -161,6 +161,8 @@ typedef struct MagtOpsTblInfo
   int Chainings;
   int SecurityErrors;
   int Errors;
+  int ConnectionsInMaxThreads;
+  int ConnectionsMaxThreadsHit;
 } MagtOpsTblInfo_t;
 
 typedef struct MagtEntriesTblInfo

ldap/servers/snmp/redhat-directory.mib

@@ -142,9 +142,18 @@ IMPORTS
 
         dsReferrals
             Counter64,
-	dsChainings
+        dsChainings
             Counter64,
 
+    -- Connection Stats
+    
+        dsConnections
+            Counter64,
+        dsMaxThreadsHit
+            Counter64,
+        dsConnectionsInMaxThreads
+            Counter64,
+             
     -- Errors
 
         dsSecurityErrors
@@ -413,6 +422,36 @@ IMPORTS
             Sections 12.4, 12.5, 12.8 & 12.9. and, RFC1777 Section 4."
         ::= {dsOpsEntry 20}
 
+    dsConnections OBJECT-TYPE
+        SYNTAX Counter64
+        MAX-ACCESS read-only
+        STATUS current
+        DESCRIPTION
+          " Number of total connections opened."
+        REFERENCE
+          " Redhat-defined 1.1"
+        ::= {dsOpsEntry 21}
+         
+    dsConnectionsInMaxThreads   OBJECT-TYPE
+        SYNTAX Counter64
+        MAX-ACCESS read-only
+        STATUS current
+        DESCRIPTION
+          " Number of connections that are currently in a max thread state."
+        REFERENCE
+          " Redhat defined 1.2."
+        ::= {dsOpsEntry 22}
+        
+    dsMaxThreadsHit OBJECT-TYPE
+        SYNTAX Counter64
+        MAX-ACCESS read-only
+        STATUS current
+        DESCRIPTION
+          " Number of times any connection has hit max threads."
+        REFERENCE
+          " Redhat defined 1.3."
+        ::= {dsOpsEntry 23} 
+        
     -- Entry statistics/Cache performance
     dsEntriesTable OBJECT-TYPE
         SYNTAX SEQUENCE OF DsEntriesEntry