windows_inc_protocol.c 63 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814
  1. /** BEGIN COPYRIGHT BLOCK
  2. * This Program is free software; you can redistribute it and/or modify it under
  3. * the terms of the GNU General Public License as published by the Free Software
  4. * Foundation; version 2 of the License.
  5. *
  6. * This Program is distributed in the hope that it will be useful, but WITHOUT
  7. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  8. * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  9. *
  10. * You should have received a copy of the GNU General Public License along with
  11. * this Program; if not, write to the Free Software Foundation, Inc., 59 Temple
  12. * Place, Suite 330, Boston, MA 02111-1307 USA.
  13. *
  14. * In addition, as a special exception, Red Hat, Inc. gives You the additional
  15. * right to link the code of this Program with code not covered under the GNU
  16. * General Public License ("Non-GPL Code") and to distribute linked combinations
  17. * including the two, subject to the limitations in this paragraph. Non-GPL Code
  18. * permitted under this exception must only link to the code of this Program
  19. * through those well defined interfaces identified in the file named EXCEPTION
  20. * found in the source code files (the "Approved Interfaces"). The files of
  21. * Non-GPL Code may instantiate templates or use macros or inline functions from
  22. * the Approved Interfaces without causing the resulting work to be covered by
  23. * the GNU General Public License. Only Red Hat, Inc. may make changes or
  24. * additions to the list of Approved Interfaces. You must obey the GNU General
  25. * Public License in all respects for all of the Program code and other code used
  26. * in conjunction with the Program except the Non-GPL Code covered by this
  27. * exception. If you modify this file, you may extend this exception to your
  28. * version of the file, but you are not obligated to do so. If you do not wish to
  29. * provide this exception without modification, you must delete this exception
  30. * statement from your version and license this file solely under the GPL without
  31. * exception.
  32. *
  33. *
  34. * Copyright (C) 2001 Sun Microsystems, Inc. Used by permission.
  35. * Copyright (C) 2005 Red Hat, Inc.
  36. * All rights reserved.
  37. * END COPYRIGHT BLOCK **/
  38. #ifdef HAVE_CONFIG_H
  39. # include <config.h>
  40. #endif
  41. /* repl5_inc_protocol.c */
  42. /*
  43. The Prot_Incremental object implements the DS 5.0 multi-master incremental
  44. replication protocol.
  45. Stuff to do:
  46. - Need to figure out how asynchronous events end up in here. They are:
  47. - entry updated in replicated area.
  48. - backoff timeout
  49. - enter/leave.
  50. Perhaps these events should be properties of the main protocol.
  51. */
  52. #include "repl.h"
  53. #include "repl5.h"
  54. #include "windowsrepl.h"
  55. #include "windows_prot_private.h"
  56. #include "slap.h" /* PSEUDO_ATTR_UNHASHED */
  57. #include "repl5_ruv.h"
  58. #include "cl5_api.h"
  59. #include "slapi-plugin.h"
  60. extern int slapi_log_urp;
  61. /*** from proto-slap.h ***/
  62. void ava_done(struct ava *ava);
  63. typedef struct windows_inc_private
  64. {
  65. char *ruv; /* RUV on remote replica (use diff type for this? - ggood */
  66. Backoff_Timer *backoff;
  67. Repl_Protocol *rp;
  68. PRLock *lock;
  69. PRUint32 eventbits;
  70. } windows_inc_private;
  71. /* Various states the incremental protocol can pass through */
  72. #define STATE_START 0 /* ONREPL - should we rename this - we don't use it just to start up? */
  73. #define STATE_WAIT_WINDOW_OPEN 1
  74. #define STATE_WAIT_CHANGES 2
  75. #define STATE_READY_TO_ACQUIRE 3
  76. #define STATE_BACKOFF_START 4 /* ONREPL - can we combine BACKOFF_START and BACKOFF states? */
  77. #define STATE_BACKOFF 5
  78. #define STATE_SENDING_UPDATES 6
  79. #define STATE_STOP_FATAL_ERROR 7
  80. #define STATE_STOP_FATAL_ERROR_PART2 8
  81. #define STATE_STOP_NORMAL_TERMINATION 9
  82. /* Events (synchronous and asynchronous; these are bits) */
  83. #define EVENT_WINDOW_OPENED 1
  84. #define EVENT_WINDOW_CLOSED 2
  85. #define EVENT_TRIGGERING_CRITERIA_MET 4 /* ONREPL - should we rename this to EVENT_CHANGE_AVAILABLE */
  86. #define EVENT_BACKOFF_EXPIRED 8
  87. #define EVENT_REPLICATE_NOW 16
  88. #define EVENT_PROTOCOL_SHUTDOWN 32
  89. #define EVENT_AGMT_CHANGED 64
  90. #define EVENT_RUN_DIRSYNC 128
  91. #define UPDATE_NO_MORE_UPDATES 201
  92. #define UPDATE_TRANSIENT_ERROR 202
  93. #define UPDATE_FATAL_ERROR 203
  94. #define UPDATE_SCHEDULE_WINDOW_CLOSED 204
  95. #define UPDATE_CONNECTION_LOST 205
  96. #define UPDATE_TIMEOUT 206
  97. #define UPDATE_YIELD 207
  98. /* Return codes from examine_update_vector */
  99. #define EXAMINE_RUV_PRISTINE_REPLICA 401
  100. #define EXAMINE_RUV_GENERATION_MISMATCH 402
  101. #define EXAMINE_RUV_REPLICA_TOO_OLD 403
  102. #define EXAMINE_RUV_OK 404
  103. #define EXAMINE_RUV_PARAM_ERROR 405
  104. #define MAX_CHANGES_PER_SESSION 10000
  105. /*
  106. * Maximum time to wait between replication sessions. If we
  107. * don't see any updates for a period equal to this interval,
  108. * we go ahead and start a replication session, just to be safe
  109. */
  110. #define MAX_WAIT_BETWEEN_SESSIONS PR_SecondsToInterval(60 * 5) /* 5 minutes */
  111. /*
  112. * tests if the protocol has been shutdown and we need to quit
  113. * event_occurred resets the bits in the bit flag, so whoever tests for shutdown
  114. * resets the flags, so the next one who tests for shutdown won't get it, so we
  115. * also look at the terminate flag
  116. */
  117. #define PROTOCOL_IS_SHUTDOWN(prp) (event_occurred(prp, EVENT_PROTOCOL_SHUTDOWN) || prp->terminate)
  118. /* Forward declarations */
  119. static PRUint32 event_occurred(Private_Repl_Protocol *prp, PRUint32 event);
  120. static void reset_events (Private_Repl_Protocol *prp);
  121. static void protocol_sleep(Private_Repl_Protocol *prp, PRIntervalTime duration);
  122. static int send_updates(Private_Repl_Protocol *prp, RUV *ruv, PRUint32 *num_changes_sent, int do_send);
  123. static void windows_inc_backoff_expired(time_t timer_fire_time, void *arg);
  124. static int windows_examine_update_vector(Private_Repl_Protocol *prp, RUV *ruv);
  125. static const char* state2name (int state);
  126. static const char* event2name (int event);
  127. static const char* acquire2name (int code);
  128. static void periodic_dirsync(time_t when, void *arg);
  129. static Slapi_Eq_Context dirsync = NULL;
  130. /*
  131. * It's specifically ok to delete a protocol instance that
  132. * is currently running. The instance will be shut down, and
  133. * then resources will be freed. Since a graceful shutdown is
  134. * attempted, this function may take some time to complete.
  135. */
  136. static void
  137. windows_inc_delete(Private_Repl_Protocol **prpp)
  138. {
  139. int rc;
  140. windows_inc_private *prp_priv = (windows_inc_private *)(*prpp)->private;
  141. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_delete\n" );
  142. /* First, stop the protocol if it isn't already stopped */
  143. /* Then, delete all resources used by the protocol */
  144. rc = slapi_eq_cancel(dirsync);
  145. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  146. "windows_inc_delete: dirsync: %p, rval: %d\n", dirsync, rc);
  147. /* if backoff is set, delete it (from EQ, as well) */
  148. if (prp_priv->backoff) {
  149. backoff_delete(&prp_priv->backoff);
  150. }
  151. if (!(*prpp)->stopped) {
  152. (*prpp)->stopped = 1;
  153. (*prpp)->stop(*prpp);
  154. }
  155. if ((*prpp)->lock) {
  156. PR_DestroyLock((*prpp)->lock);
  157. (*prpp)->lock = NULL;
  158. }
  159. if ((*prpp)->cvar) {
  160. PR_DestroyCondVar((*prpp)->cvar);
  161. (*prpp)->cvar = NULL;
  162. }
  163. slapi_ch_free((void **)&(*prpp)->private);
  164. slapi_ch_free((void **)prpp);
  165. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_delete\n" );
  166. }
  167. /* helper function */
  168. void
  169. w_set_pause_and_busy_time(Private_Repl_Protocol *prp, long *pausetime, long *busywaittime)
  170. {
  171. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> w_set_pause_and_busy_time\n" );
  172. /* If neither are set, set busy time to its default */
  173. if (!*pausetime && !*busywaittime)
  174. {
  175. *busywaittime = repl5_get_backoff_min(prp);
  176. }
  177. /* pause time must be at least 1 more than the busy backoff time */
  178. if (*pausetime && !*busywaittime)
  179. {
  180. /*
  181. * user specified a pause time but no busy wait time - must
  182. * set busy wait time to 1 less than pause time - if pause
  183. * time is 1, we must set it to 2
  184. */
  185. if (*pausetime < 2)
  186. {
  187. *pausetime = 2;
  188. }
  189. *busywaittime = *pausetime - 1;
  190. }
  191. else if (!*pausetime && *busywaittime)
  192. {
  193. /*
  194. * user specified a busy wait time but no pause time - must
  195. * set pause time to 1 more than busy wait time
  196. */
  197. *pausetime = *busywaittime + 1;
  198. }
  199. else if (*pausetime && *busywaittime && *pausetime <= *busywaittime)
  200. {
  201. /*
  202. * user specified both pause and busy wait times, but the pause
  203. * time was <= busy wait time - pause time must be at least
  204. * 1 more than the busy wait time
  205. */
  206. *pausetime = *busywaittime + 1;
  207. }
  208. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= w_set_pause_and_busy_time\n" );
  209. }
  210. /*
  211. * Do the incremental protocol.
  212. *
  213. * What's going on here? This thing is a state machine. It has the
  214. * following states:
  215. *
  216. * State transition table:
  217. *
  218. * Curr State Condition/Event Next State
  219. * ---------- ------------ -----------
  220. * START schedule window is open ACQUIRE_REPLICA
  221. * schedule window is closed WAIT_WINDOW_OPEN
  222. * WAIT_WINDOW_OPEN schedule change START
  223. * replicate now ACQUIRE_REPLICA
  224. * schedule window opens ACQUIRE_REPLICA
  225. * ACQUIRE_REPLICA acquired replica SEND_CHANGES
  226. * failed to acquire - transient error START_BACKOFF
  227. * failed to acquire - fatal error STOP_FATAL_ERROR
  228. * SEND_CHANGES can't update CONSUMER_NEEDS_REINIT
  229. * no changes to send WAIT_CHANGES
  230. * can't send - thransient error START_BACKOF
  231. * can't send - window closed WAIT_WINDOW_OPEN
  232. * can'r send - fatal error STOP_FATAL_ERROR
  233. * START_BACKOF replicate now ACQUIRE_REPLICA
  234. * schedule changes START
  235. * schedule window closes WAIT_WINDOW_OPEN
  236. * backoff expires & can acquire SEND_CHANGES
  237. * backoff expires & can't acquire-trans BACKOFF
  238. * backoff expires & can't acquire-fatal STOP_FATAL_ERROR
  239. * BACKOF replicate now ACQUIRE_REPLICA
  240. * schedule changes START
  241. * schedule window closes WAIT_WINDOW_OPEN
  242. * backoff expires & can acquire SEND_CHANGES
  243. * backoff expires & can't acquire-trans BACKOFF
  244. * backoff expires & can't acquire-fatal STOP_FATAL_ERROR
  245. * WAIT_CHANGES schedule window closes WAIT_WINDOW_OPEN
  246. * replicate_now ACQUIRE_REPLICA
  247. * change available ACQUIRE_REPLICA
  248. * schedule_change START
  249. */
  250. /*
  251. * DBDB: what follows is quite possibly the worst code I have ever seen.
  252. * Unfortunately we chose not to re-write it when we did the windows sync version.
  253. */
  254. /*
  255. * Main state machine for the incremental protocol. This routine will,
  256. * under normal circumstances, not return until the protocol is shut
  257. * down.
  258. */
  259. static void
  260. windows_inc_run(Private_Repl_Protocol *prp)
  261. {
  262. int current_state = STATE_START;
  263. int next_state = STATE_START;
  264. windows_inc_private *prp_priv = (windows_inc_private *)prp->private;
  265. int done = 0;
  266. int e1 = 0;
  267. RUV *ruv = NULL;
  268. Replica *replica = NULL;
  269. int wait_change_timer_set = 0;
  270. time_t last_start_time = 0;
  271. PRUint32 num_changes_sent = 0;
  272. /* use a different backoff timer strategy for ACQUIRE_REPLICA_BUSY errors */
  273. PRBool use_busy_backoff_timer = PR_FALSE;
  274. long pausetime = 0;
  275. long busywaittime = 0;
  276. unsigned long current_interval = 0;
  277. unsigned long interval = 0;
  278. int one_way;
  279. PRBool run_dirsync = PR_FALSE;
  280. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_run\n" );
  281. prp->stopped = 0;
  282. prp->terminate = 0;
  283. windows_private_load_dirsync_cookie(prp->agmt);
  284. do {
  285. int rc = 0;
  286. one_way = windows_private_get_one_way(prp->agmt);
  287. /* Take action, based on current state, and compute new state. */
  288. switch (current_state)
  289. {
  290. case STATE_START:
  291. dev_debug("windows_inc_run(STATE_START)");
  292. if (PROTOCOL_IS_SHUTDOWN(prp))
  293. {
  294. done = 1;
  295. break;
  296. }
  297. /*
  298. * Our initial state. See if we're in a schedule window. If
  299. * so, then we're ready to acquire the replica and see if it
  300. * needs any updates from us. If not, then wait for the window
  301. * to open.
  302. */
  303. if (agmt_schedule_in_window_now(prp->agmt))
  304. {
  305. next_state = STATE_READY_TO_ACQUIRE;
  306. } else
  307. {
  308. next_state = STATE_WAIT_WINDOW_OPEN;
  309. }
  310. /* we can get here from other states because some events happened and were
  311. not cleared. For instance when we wake up in STATE_WAIT_CHANGES state.
  312. Since this is a fresh start state, we should clear all events */
  313. /* ONREPL - this does not feel right - we should take another look
  314. at this state machine */
  315. reset_events (prp);
  316. /* Cancel any linger timer that might be in effect... */
  317. windows_conn_cancel_linger(prp->conn);
  318. /* ... and disconnect, if currently connected */
  319. windows_conn_disconnect(prp->conn);
  320. /* get the new pause time, if any */
  321. pausetime = agmt_get_pausetime(prp->agmt);
  322. /* get the new busy wait time, if any */
  323. busywaittime = agmt_get_busywaittime(prp->agmt);
  324. if (pausetime || busywaittime)
  325. {
  326. /* helper function to make sure they are set correctly */
  327. w_set_pause_and_busy_time(prp, &pausetime, &busywaittime);
  328. }
  329. /* Check if the interval changed */
  330. interval = windows_private_get_sync_interval(prp->agmt) * 1000;
  331. if(interval != current_interval){
  332. current_interval = interval;
  333. if(dirsync){
  334. int rc = slapi_eq_cancel(dirsync);
  335. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  336. "windows_inc_runs: cancelled dirsync: %p, rval: %d\n",
  337. dirsync, rc);
  338. }
  339. dirsync = slapi_eq_repeat(periodic_dirsync, (void*) prp, (time_t)0 , interval);
  340. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  341. "windows_inc_runs: new dirsync: %p\n", dirsync);
  342. }
  343. break;
  344. case STATE_WAIT_WINDOW_OPEN:
  345. /*
  346. * We're waiting for a schedule window to open. If one did,
  347. * or we receive a "replicate now" event, then start a protocol
  348. * session immediately. If the replication schedule changed, go
  349. * back to start. Otherwise, go back to sleep.
  350. */
  351. dev_debug("windows_inc_run(STATE_WAIT_WINDOW_OPEN)");
  352. if (PROTOCOL_IS_SHUTDOWN(prp))
  353. {
  354. done = 1;
  355. break;
  356. }
  357. else if (event_occurred(prp, EVENT_WINDOW_OPENED))
  358. {
  359. next_state = STATE_READY_TO_ACQUIRE;
  360. }
  361. else if (event_occurred(prp, EVENT_REPLICATE_NOW))
  362. {
  363. next_state = STATE_READY_TO_ACQUIRE;
  364. }
  365. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  366. {
  367. next_state = STATE_START;
  368. run_dirsync = PR_TRUE;
  369. windows_conn_set_agmt_changed(prp->conn);
  370. }
  371. else if (event_occurred(prp, EVENT_TRIGGERING_CRITERIA_MET)) /* change available */
  372. {
  373. /* just ignore it and go to sleep */
  374. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  375. }
  376. else if ((e1 = event_occurred(prp, EVENT_WINDOW_CLOSED)) ||
  377. event_occurred(prp, EVENT_BACKOFF_EXPIRED))
  378. {
  379. /* this events - should not occur - log a warning and go to sleep */
  380. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  381. "%s: Incremental protocol: "
  382. "event %s should not occur in state %s; going to sleep\n",
  383. agmt_get_long_name(prp->agmt),
  384. e1 ? event2name(EVENT_WINDOW_CLOSED) : event2name(EVENT_BACKOFF_EXPIRED),
  385. state2name(current_state));
  386. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  387. }
  388. else if (event_occurred(prp, EVENT_RUN_DIRSYNC)) /* periodic_dirsync */
  389. {
  390. /* just ignore it and go to sleep */
  391. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  392. }
  393. else
  394. {
  395. /* wait until window opens or an event occurs */
  396. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  397. "%s: Incremental protocol: "
  398. "waiting for update window to open\n", agmt_get_long_name(prp->agmt));
  399. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  400. }
  401. break;
  402. case STATE_WAIT_CHANGES:
  403. /*
  404. * We're in a replication window, but we're waiting for more
  405. * changes to accumulate before we actually hook up and send
  406. * them.
  407. */
  408. dev_debug("windows_inc_run(STATE_WAIT_CHANGES)");
  409. if (PROTOCOL_IS_SHUTDOWN(prp))
  410. {
  411. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): PROTOCOL_IS_SHUTING_DOWN -> end windows_inc_run\n");
  412. done = 1;
  413. break;
  414. }
  415. else if (event_occurred(prp, EVENT_REPLICATE_NOW))
  416. {
  417. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_REPLICATE_NOW received -> STATE_READY_TO_ACQUIRE\n");
  418. next_state = STATE_READY_TO_ACQUIRE;
  419. wait_change_timer_set = 0;
  420. /* We also want to run dirsync on a 'replicate now' event */
  421. run_dirsync = PR_TRUE;
  422. }
  423. else if ( event_occurred(prp, EVENT_RUN_DIRSYNC))
  424. {
  425. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_REPLICATE_NOW received -> STATE_READY_TO_ACQUIRE\n");
  426. next_state = STATE_READY_TO_ACQUIRE;
  427. wait_change_timer_set = 0;
  428. run_dirsync = PR_TRUE;
  429. }
  430. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  431. {
  432. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_AGMT_CHANGED received -> STATE_START\n");
  433. next_state = STATE_START;
  434. windows_conn_set_agmt_changed(prp->conn);
  435. wait_change_timer_set = 0;
  436. /* We also want to run dirsync on a 'agreement changed' event, because that's how we receive 'send updates now' */
  437. run_dirsync = PR_TRUE;
  438. }
  439. else if (event_occurred(prp, EVENT_WINDOW_CLOSED))
  440. {
  441. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_WINDOW_CLOSED received -> STATE_WAIT_WINDOW_OPEN\n");
  442. next_state = STATE_WAIT_WINDOW_OPEN;
  443. wait_change_timer_set = 0;
  444. }
  445. else if (event_occurred(prp, EVENT_TRIGGERING_CRITERIA_MET) )
  446. {
  447. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_TRIGGERING_CRITERIA_MET received -> STATE_READY_TO_ACQUIRE\n");
  448. next_state = STATE_READY_TO_ACQUIRE;
  449. wait_change_timer_set = 0;
  450. /* We want to run dirsync to catch values generated by AD after
  451. * it process operations that we send to it (such as GUID values). */
  452. run_dirsync = PR_TRUE;
  453. }
  454. else if ((e1 = event_occurred(prp, EVENT_WINDOW_OPENED)) ||
  455. event_occurred(prp, EVENT_BACKOFF_EXPIRED))
  456. {
  457. /* this events - should not occur - log a warning and clear the event */
  458. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name, "%s: Incremental protocol: "
  459. "event %s should not occur in state %s\n",
  460. agmt_get_long_name(prp->agmt),
  461. e1 ? event2name(EVENT_WINDOW_OPENED) : event2name(EVENT_BACKOFF_EXPIRED),
  462. state2name(current_state));
  463. wait_change_timer_set = 0;
  464. }
  465. else
  466. {
  467. if (wait_change_timer_set)
  468. {
  469. /* We are here because our timer expired */
  470. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): wait_change_timer_set expired -> STATE_START\n");
  471. next_state = STATE_START;
  472. run_dirsync = PR_TRUE;
  473. wait_change_timer_set = 0;
  474. }
  475. else
  476. {
  477. /* We are here because the last replication session
  478. * finished or aborted.
  479. */
  480. wait_change_timer_set = 1;
  481. protocol_sleep(prp, MAX_WAIT_BETWEEN_SESSIONS);
  482. }
  483. }
  484. break;
  485. case STATE_READY_TO_ACQUIRE:
  486. dev_debug("windows_inc_run(STATE_READY_TO_ACQUIRE)");
  487. if (PROTOCOL_IS_SHUTDOWN(prp))
  488. {
  489. done = 1;
  490. break;
  491. }
  492. /* ONREPL - at this state we unconditionally acquire the replica
  493. ignoring all events. Not sure if this is good */
  494. object_acquire(prp->replica_object);
  495. rc = windows_acquire_replica(prp, &ruv , (run_dirsync == 0) /* yes, check the consumer RUV for incremental, but not if we're going to dirsync afterwards */);
  496. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  497. "windows_acquire_replica returned %s (%d)\n",
  498. acquire2name(rc),
  499. rc);
  500. use_busy_backoff_timer = PR_FALSE; /* default */
  501. if (rc == ACQUIRE_SUCCESS)
  502. {
  503. next_state = STATE_SENDING_UPDATES;
  504. }
  505. else if (rc == ACQUIRE_REPLICA_BUSY)
  506. {
  507. next_state = STATE_BACKOFF_START;
  508. use_busy_backoff_timer = PR_TRUE;
  509. }
  510. else if (rc == ACQUIRE_CONSUMER_WAS_UPTODATE)
  511. {
  512. next_state = STATE_WAIT_CHANGES;
  513. }
  514. else if (rc == ACQUIRE_TRANSIENT_ERROR)
  515. {
  516. next_state = STATE_BACKOFF_START;
  517. }
  518. else if (rc == ACQUIRE_FATAL_ERROR)
  519. {
  520. next_state = STATE_STOP_FATAL_ERROR;
  521. }
  522. if (rc != ACQUIRE_SUCCESS)
  523. {
  524. int optype, ldaprc;
  525. windows_conn_get_error(prp->conn, &optype, &ldaprc);
  526. agmt_set_last_update_status(prp->agmt, ldaprc,
  527. prp->last_acquire_response_code, NULL);
  528. }
  529. object_release(prp->replica_object);
  530. break;
  531. case STATE_BACKOFF_START:
  532. dev_debug("windows_inc_run(STATE_BACKOFF_START)");
  533. if (PROTOCOL_IS_SHUTDOWN(prp))
  534. {
  535. done = 1;
  536. break;
  537. }
  538. if (event_occurred(prp, EVENT_REPLICATE_NOW) || event_occurred(prp, EVENT_RUN_DIRSYNC))
  539. {
  540. next_state = STATE_READY_TO_ACQUIRE;
  541. }
  542. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  543. {
  544. next_state = STATE_START;
  545. run_dirsync = PR_TRUE; /* Also trigger dirsync for the 'send updates now' feature */
  546. windows_conn_set_agmt_changed(prp->conn);
  547. }
  548. else if (event_occurred (prp, EVENT_WINDOW_CLOSED))
  549. {
  550. next_state = STATE_WAIT_WINDOW_OPEN;
  551. }
  552. else if (event_occurred (prp, EVENT_TRIGGERING_CRITERIA_MET))
  553. {
  554. /* consume and ignore */
  555. }
  556. else if ((e1 = event_occurred (prp, EVENT_WINDOW_OPENED)) ||
  557. event_occurred (prp, EVENT_BACKOFF_EXPIRED))
  558. {
  559. /* This should never happen */
  560. /* this events - should not occur - log a warning and go to sleep */
  561. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  562. "%s: Incremental protocol: event %s should not occur in state %s\n",
  563. agmt_get_long_name(prp->agmt),
  564. e1 ? event2name(EVENT_WINDOW_OPENED) : event2name(EVENT_BACKOFF_EXPIRED),
  565. state2name(current_state));
  566. }
  567. else
  568. {
  569. /* Set up the backoff timer to wake us up at the appropriate time */
  570. /* if previous backoff set up, delete it. */
  571. if (prp_priv->backoff) {
  572. backoff_delete(&prp_priv->backoff);
  573. }
  574. if (use_busy_backoff_timer)
  575. {
  576. /* we received a busy signal from the consumer, wait for a while */
  577. if (!busywaittime){
  578. busywaittime = repl5_get_backoff_min(prp);
  579. }
  580. prp_priv->backoff = backoff_new(BACKOFF_FIXED, busywaittime, busywaittime);
  581. }
  582. else
  583. {
  584. prp_priv->backoff = backoff_new(BACKOFF_EXPONENTIAL, repl5_get_backoff_min(prp),
  585. repl5_get_backoff_max(prp) );
  586. }
  587. next_state = STATE_BACKOFF;
  588. backoff_reset(prp_priv->backoff, windows_inc_backoff_expired, (void *)prp);
  589. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  590. use_busy_backoff_timer = PR_FALSE;
  591. }
  592. break;
  593. case STATE_BACKOFF:
  594. /*
  595. * We're in a backoff state.
  596. */
  597. dev_debug("windows_inc_run(STATE_BACKOFF)");
  598. if (PROTOCOL_IS_SHUTDOWN(prp))
  599. {
  600. if (prp_priv->backoff)
  601. backoff_delete(&prp_priv->backoff);
  602. done = 1;
  603. break;
  604. }
  605. else if (event_occurred(prp, EVENT_REPLICATE_NOW) || event_occurred(prp, EVENT_RUN_DIRSYNC))
  606. {
  607. next_state = STATE_READY_TO_ACQUIRE;
  608. }
  609. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  610. {
  611. next_state = STATE_START;
  612. run_dirsync = PR_TRUE;
  613. windows_conn_set_agmt_changed(prp->conn);
  614. /* Destroy the backoff timer, since we won't need it anymore */
  615. if (prp_priv->backoff)
  616. backoff_delete(&prp_priv->backoff);
  617. }
  618. else if (event_occurred(prp, EVENT_WINDOW_CLOSED))
  619. {
  620. next_state = STATE_WAIT_WINDOW_OPEN;
  621. /* Destroy the backoff timer, since we won't need it anymore */
  622. if (prp_priv->backoff)
  623. backoff_delete(&prp_priv->backoff);
  624. }
  625. else if (event_occurred(prp, EVENT_BACKOFF_EXPIRED))
  626. {
  627. rc = windows_acquire_replica(prp, &ruv, 1 /* check RUV for incremental */);
  628. use_busy_backoff_timer = PR_FALSE;
  629. if (rc == ACQUIRE_SUCCESS)
  630. {
  631. next_state = STATE_SENDING_UPDATES;
  632. }
  633. else if (rc == ACQUIRE_REPLICA_BUSY)
  634. {
  635. next_state = STATE_BACKOFF;
  636. use_busy_backoff_timer = PR_TRUE;
  637. }
  638. else if (rc == ACQUIRE_CONSUMER_WAS_UPTODATE)
  639. {
  640. next_state = STATE_WAIT_CHANGES;
  641. }
  642. else if (rc == ACQUIRE_TRANSIENT_ERROR)
  643. {
  644. next_state = STATE_BACKOFF;
  645. }
  646. else if (rc == ACQUIRE_FATAL_ERROR)
  647. {
  648. next_state = STATE_STOP_FATAL_ERROR;
  649. }
  650. if (rc != ACQUIRE_SUCCESS)
  651. {
  652. int optype, ldaprc;
  653. windows_conn_get_error(prp->conn, &optype, &ldaprc);
  654. agmt_set_last_update_status(prp->agmt, ldaprc,
  655. prp->last_acquire_response_code, NULL);
  656. }
  657. /*
  658. * We either need to step the backoff timer, or
  659. * destroy it if we don't need it anymore.
  660. */
  661. if (STATE_BACKOFF == next_state)
  662. {
  663. time_t next_fire_time;
  664. time_t now;
  665. /* Step the backoff timer */
  666. time(&now);
  667. next_fire_time = backoff_step(prp_priv->backoff);
  668. /* And go back to sleep */
  669. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  670. "%s: Replication session backing off for %ld seconds\n",
  671. agmt_get_long_name(prp->agmt),
  672. next_fire_time - now);
  673. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  674. }
  675. else
  676. {
  677. /* Destroy the backoff timer, since we won't need it anymore */
  678. backoff_delete(&prp_priv->backoff);
  679. }
  680. }
  681. else if (event_occurred(prp, EVENT_TRIGGERING_CRITERIA_MET))
  682. {
  683. /* changes are available */
  684. if ( prp_priv->backoff == NULL || backoff_expired (prp_priv->backoff, 60) )
  685. {
  686. /*
  687. * Have seen cases that the agmt stuck here forever since
  688. * somehow the backoff timer was not in event queue anymore.
  689. * If the backoff timer has expired more than 60 seconds,
  690. * destroy it.
  691. */
  692. if ( prp_priv->backoff )
  693. backoff_delete(&prp_priv->backoff);
  694. next_state = STATE_READY_TO_ACQUIRE;
  695. }
  696. else
  697. {
  698. /* ignore changes and go to sleep */
  699. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  700. }
  701. }
  702. else if (event_occurred(prp, EVENT_WINDOW_OPENED))
  703. {
  704. /* this should never happen - log an error and go to sleep */
  705. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name, "%s: Incremental protocol: "
  706. "event %s should not occur in state %s; going to sleep\n",
  707. agmt_get_long_name(prp->agmt),
  708. event2name(EVENT_WINDOW_OPENED), state2name(current_state));
  709. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  710. }
  711. break;
  712. case STATE_SENDING_UPDATES:
  713. dev_debug("windows_inc_run(STATE_SENDING_UPDATES)");
  714. agmt_set_update_in_progress(prp->agmt, PR_TRUE);
  715. num_changes_sent = 0;
  716. last_start_time = current_time();
  717. agmt_set_last_update_start(prp->agmt, last_start_time);
  718. /*
  719. * We've acquired the replica, and are ready to send any
  720. * needed updates.
  721. */
  722. if (PROTOCOL_IS_SHUTDOWN(prp))
  723. {
  724. windows_release_replica (prp);
  725. done = 1;
  726. agmt_set_update_in_progress(prp->agmt, PR_FALSE);
  727. agmt_set_last_update_end(prp->agmt, current_time());
  728. /* MAB: I don't find the following status correct. How do we know it has
  729. been stopped by an admin and not by a total update request, for instance?
  730. In any case, how is this protocol shutdown situation different from all the
  731. other ones that are present in this state machine? */
  732. /* richm: We at least need to let monitors know that the protocol has been
  733. shutdown - maybe they can figure out why */
  734. agmt_set_last_update_status(prp->agmt, 0, 0, "Protocol stopped");
  735. agmt_update_done(prp->agmt, 0);
  736. break;
  737. }
  738. agmt_set_last_update_status(prp->agmt, 0, 0, "Incremental update started");
  739. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> windows_examine_update_vector");
  740. rc = windows_examine_update_vector(prp, ruv);
  741. /*
  742. * Decide what to do next - proceed with incremental,
  743. * backoff, or total update
  744. */
  745. switch (rc)
  746. {
  747. case EXAMINE_RUV_PARAM_ERROR:
  748. /* this is really bad - we have NULL prp! */
  749. next_state = STATE_STOP_FATAL_ERROR;
  750. break;
  751. case EXAMINE_RUV_PRISTINE_REPLICA:
  752. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  753. "%s: Replica has no update vector. It has never been initialized.\n",
  754. agmt_get_long_name(prp->agmt));
  755. next_state = STATE_BACKOFF_START;
  756. break;
  757. case EXAMINE_RUV_GENERATION_MISMATCH:
  758. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  759. "%s: The remote replica has a different database generation ID than "
  760. "the local database. You may have to reinitialize the remote replica, "
  761. "or the local replica.\n", agmt_get_long_name(prp->agmt));
  762. next_state = STATE_BACKOFF_START;
  763. break;
  764. case EXAMINE_RUV_REPLICA_TOO_OLD:
  765. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  766. "%s: Replica update vector is too out of date to bring "
  767. "into sync using the incremental protocol. The replica "
  768. "must be reinitialized.\n", agmt_get_long_name(prp->agmt));
  769. next_state = STATE_BACKOFF_START;
  770. break;
  771. case EXAMINE_RUV_OK:
  772. /* update our csn generator state with the consumer's ruv data */
  773. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> windows_examine_update_vector OK");
  774. object_acquire(prp->replica_object);
  775. replica = object_get_data(prp->replica_object);
  776. rc = replica_update_csngen_state (replica, ruv);
  777. object_release (prp->replica_object);
  778. replica = NULL;
  779. if (rc == CSN_LIMIT_EXCEEDED) /* too much skew */
  780. {
  781. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  782. "%s: Incremental protocol: fatal error - too much time skew between replicas!\n",
  783. agmt_get_long_name(prp->agmt));
  784. next_state = STATE_STOP_FATAL_ERROR;
  785. }
  786. else if (rc != 0) /* internal error */
  787. {
  788. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  789. "%s: Incremental protocol: fatal internal error updating the CSN generator!\n",
  790. agmt_get_long_name(prp->agmt));
  791. next_state = STATE_STOP_FATAL_ERROR;
  792. }
  793. else
  794. {
  795. /* call begin incremental update callback */
  796. winsync_plugin_call_begin_update_cb(prp->agmt,
  797. windows_private_get_directory_subtree(prp->agmt),
  798. windows_private_get_windows_subtree(prp->agmt),
  799. 0 /* is_total == FALSE */);
  800. if ((one_way == ONE_WAY_SYNC_DISABLED) || (one_way == ONE_WAY_SYNC_TO_AD)) {
  801. rc = send_updates(prp, ruv, &num_changes_sent, 1 /* actually send updates */);
  802. } else {
  803. /* We call send_updates to fast-forward the RUV
  804. * without actually sending any updates. */
  805. rc = send_updates(prp, ruv, &num_changes_sent, 0 /* don't send updates */);
  806. }
  807. if (rc == UPDATE_NO_MORE_UPDATES)
  808. {
  809. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_NO_MORE_UPDATES -> STATE_WAIT_CHANGES");
  810. agmt_set_last_update_status(prp->agmt, 0, 0, "Incremental update succeeded");
  811. next_state = STATE_WAIT_CHANGES;
  812. }
  813. else if (rc == UPDATE_YIELD)
  814. {
  815. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_YIELD -> STATE_BACKOFF_START");
  816. agmt_set_last_update_status(prp->agmt, 0, 0, "Incremental update succeeded and yielded");
  817. next_state = STATE_BACKOFF_START;
  818. }
  819. else if (rc == UPDATE_TRANSIENT_ERROR)
  820. {
  821. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_TRANSIENT_ERROR -> STATE_BACKOFF_START");
  822. next_state = STATE_BACKOFF_START;
  823. }
  824. else if (rc == UPDATE_FATAL_ERROR)
  825. {
  826. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_FATAL_ERROR -> STATE_STOP_FATAL_ERROR");
  827. next_state = STATE_STOP_FATAL_ERROR;
  828. }
  829. else if (rc == UPDATE_SCHEDULE_WINDOW_CLOSED)
  830. {
  831. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_SCHEDULE_WINDOW_CLOSED -> STATE_WAIT_WINDOW_OPEN");
  832. /* ONREPL - I don't think we should check this. We might be
  833. here because of replicate_now event - so we don't care
  834. about the schedule */
  835. next_state = STATE_WAIT_WINDOW_OPEN;
  836. /* ONREPL - do we need to release the replica here ? */
  837. windows_conn_disconnect (prp->conn);
  838. }
  839. else if (rc == UPDATE_CONNECTION_LOST)
  840. {
  841. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_CONNECTION_LOST -> STATE_BACKOFF_START");
  842. next_state = STATE_BACKOFF_START;
  843. }
  844. else if (rc == UPDATE_TIMEOUT)
  845. {
  846. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_TIMEOUT -> STATE_BACKOFF_START");
  847. next_state = STATE_BACKOFF_START;
  848. }
  849. }
  850. last_start_time = 0UL;
  851. break;
  852. }
  853. if ( run_dirsync && ((one_way == ONE_WAY_SYNC_DISABLED) || (one_way == ONE_WAY_SYNC_FROM_AD)))
  854. {
  855. /* Don't run dirsync if we encountered
  856. * an error when sending updates to AD. */
  857. if (rc == UPDATE_NO_MORE_UPDATES) {
  858. windows_dirsync_inc_run(prp);
  859. windows_private_save_dirsync_cookie(prp->agmt);
  860. }
  861. run_dirsync = PR_FALSE;
  862. }
  863. agmt_set_last_update_end(prp->agmt, current_time());
  864. agmt_set_update_in_progress(prp->agmt, PR_FALSE);
  865. agmt_update_done(prp->agmt, 0);
  866. /* If timed out, close the connection after released the replica */
  867. windows_release_replica(prp);
  868. if (rc == UPDATE_TIMEOUT) {
  869. windows_conn_disconnect(prp->conn);
  870. }
  871. /* call end incremental update callback */
  872. winsync_plugin_call_end_update_cb(prp->agmt,
  873. windows_private_get_directory_subtree(prp->agmt),
  874. windows_private_get_windows_subtree(prp->agmt),
  875. 0 /* is_total == FALSE */);
  876. if (rc == UPDATE_NO_MORE_UPDATES && num_changes_sent > 0)
  877. {
  878. if (pausetime > 0)
  879. {
  880. /* richm - 20020219 - If we have acquired the consumer, and another master has gone
  881. into backoff waiting for us to release it, we may acquire the replica sooner
  882. than the other master has a chance to, and the other master may not be able
  883. to acquire the consumer for a long time (hours, days?) if this server is
  884. under a heavy load (see reliab06 et. al. system tests)
  885. So, this sleep gives the other master(s) a chance to acquire the consumer
  886. replica */
  887. long loops = pausetime;
  888. /* the while loop is so that we don't just sleep and sleep if an
  889. event comes in that we should handle immediately (like shutdown) */
  890. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  891. "%s: Pausing updates for %ld seconds to allow other suppliers to update consumer\n",
  892. agmt_get_long_name(prp->agmt), pausetime);
  893. while (loops-- && !(PROTOCOL_IS_SHUTDOWN(prp)))
  894. {
  895. DS_Sleep(PR_SecondsToInterval(1));
  896. }
  897. }
  898. else if (num_changes_sent > 10)
  899. {
  900. /* wait for consumer to write its ruv if the replication was busy */
  901. /* When asked, consumer sends its ruv in cache to the supplier. */
  902. /* DS_Sleep ( PR_SecondsToInterval(1) ); */
  903. }
  904. }
  905. break;
  906. case STATE_STOP_FATAL_ERROR:
  907. /*
  908. * We encountered some sort of a fatal error. Suspend.
  909. */
  910. /* XXXggood update state in replica */
  911. agmt_set_last_update_status(prp->agmt, -1, 0, "Incremental update has failed and requires administrator action");
  912. dev_debug("windows_inc_run(STATE_STOP_FATAL_ERROR)");
  913. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  914. "%s: Incremental update failed and requires administrator action\n",
  915. agmt_get_long_name(prp->agmt));
  916. next_state = STATE_STOP_FATAL_ERROR_PART2;
  917. break;
  918. case STATE_STOP_FATAL_ERROR_PART2:
  919. if (PROTOCOL_IS_SHUTDOWN(prp))
  920. {
  921. done = 1;
  922. break;
  923. }
  924. /* MAB: This state is the FATAL state where we are supposed to get
  925. as a result of a FATAL error on send_updates. But, as bug
  926. states, send_updates was always returning TRANSIENT errors and never
  927. FATAL... In other words, this code has never been tested before...
  928. As of 01/16/01, this piece of code was in a very dangerous state. In particular,
  929. 1) it does not catch any events
  930. 2) it is a terminal state (once reached it never transitions to a different state)
  931. Both things combined make this state to become a consuming infinite loop
  932. that is useless after all (we are in a fatal place requiring manual admin jobs */
  933. /* MAB: The following lines fix problem number 1 above... When the code gets
  934. into this state, it should only get a chance to get out of it by an
  935. EVENT_AGMT_CHANGED event... All other events should be ignored */
  936. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  937. {
  938. dev_debug("windows_inc_run(STATE_STOP_FATAL_ERROR): EVENT_AGMT_CHANGED received\n");
  939. /* Chance to recover for the EVENT_AGMT_CHANGED event.
  940. This is not mandatory, but fixes problem 2 above */
  941. next_state = STATE_STOP_NORMAL_TERMINATION;
  942. }
  943. else
  944. {
  945. dev_debug("windows_inc_run(STATE_STOP_FATAL_ERROR): Event received. Clearing it\n");
  946. reset_events (prp);
  947. }
  948. protocol_sleep (prp, PR_INTERVAL_NO_TIMEOUT);
  949. break;
  950. case STATE_STOP_NORMAL_TERMINATION:
  951. /*
  952. * We encountered some sort of a fatal error. Return.
  953. */
  954. /* XXXggood update state in replica */
  955. dev_debug("windows_inc_run(STATE_STOP_NORMAL_TERMINATION)");
  956. done = 1;
  957. break;
  958. }
  959. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  960. "%s: State: %s -> %s\n",
  961. agmt_get_long_name(prp->agmt),
  962. state2name(current_state), state2name(next_state));
  963. current_state = next_state;
  964. } while (!done);
  965. /* remove_protocol_callbacks(prp); */
  966. prp->stopped = 1;
  967. /* Cancel any linger timer that might be in effect... */
  968. windows_conn_cancel_linger(prp->conn);
  969. /* ... and disconnect, if currently connected */
  970. windows_conn_disconnect(prp->conn);
  971. ruv_destroy ( &ruv );
  972. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_run\n" );
  973. }
  974. /*
  975. * Go to sleep until awakened.
  976. */
  977. static void
  978. protocol_sleep(Private_Repl_Protocol *prp, PRIntervalTime duration)
  979. {
  980. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> protocol_sleep\n" );
  981. PR_ASSERT(NULL != prp);
  982. PR_Lock(prp->lock);
  983. /* we should not go to sleep if there are events available to be processed.
  984. Otherwise, we can miss the event that suppose to wake us up */
  985. if (prp->eventbits == 0)
  986. PR_WaitCondVar(prp->cvar, duration);
  987. else
  988. {
  989. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  990. "%s: Incremental protocol: can't go to sleep: event bits - %x\n",
  991. agmt_get_long_name(prp->agmt), prp->eventbits);
  992. }
  993. PR_Unlock(prp->lock);
  994. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= protocol_sleep\n" );
  995. }
  996. /*
  997. * Notify the protocol about some event. Signal the condition
  998. * variable in case the protocol is sleeping. Multiple occurences
  999. * of a single event type are not remembered (e.g. no stack
  1000. * of events is maintained).
  1001. */
  1002. static void
  1003. event_notify(Private_Repl_Protocol *prp, PRUint32 event)
  1004. {
  1005. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> event_notify\n" );
  1006. PR_ASSERT(NULL != prp);
  1007. PR_Lock(prp->lock);
  1008. prp->eventbits |= event;
  1009. PR_NotifyCondVar(prp->cvar);
  1010. PR_Unlock(prp->lock);
  1011. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= event_notify\n" );
  1012. }
  1013. /*
  1014. * Test to see if an event occurred. The event is cleared when
  1015. * read.
  1016. */
  1017. static PRUint32
  1018. event_occurred(Private_Repl_Protocol *prp, PRUint32 event)
  1019. {
  1020. PRUint32 return_value;
  1021. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> event_occurred\n" );
  1022. PR_ASSERT(NULL != prp);
  1023. PR_Lock(prp->lock);
  1024. return_value = (prp->eventbits & event);
  1025. prp->eventbits &= ~event; /* Clear event */
  1026. PR_Unlock(prp->lock);
  1027. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= event_occurred\n" );
  1028. return return_value;
  1029. }
  1030. static void
  1031. reset_events (Private_Repl_Protocol *prp)
  1032. {
  1033. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> reset_events\n" );
  1034. PR_ASSERT(NULL != prp);
  1035. PR_Lock(prp->lock);
  1036. prp->eventbits = 0;
  1037. PR_Unlock(prp->lock);
  1038. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= reset_events\n" );
  1039. }
  1040. static PRBool
  1041. is_dummy_operation (const slapi_operation_parameters *op)
  1042. {
  1043. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> is_dummy_operation\n" );
  1044. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= is_dummy_operation\n" );
  1045. return (strcmp (op->target_address.uniqueid, START_ITERATION_ENTRY_UNIQUEID) == 0);
  1046. }
  1047. void
  1048. w_cl5_operation_parameters_done (struct slapi_operation_parameters *sop)
  1049. {
  1050. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> w_cl5_operation_parameters_done\n" );
  1051. if(sop!=NULL) {
  1052. switch(sop->operation_type)
  1053. {
  1054. case SLAPI_OPERATION_BIND:
  1055. slapi_ch_free((void **)&(sop->p.p_bind.bind_saslmechanism));
  1056. if (sop->p.p_bind.bind_creds)
  1057. ber_bvecfree((struct berval**)&(sop->p.p_bind.bind_creds));
  1058. if (sop->p.p_bind.bind_ret_saslcreds)
  1059. ber_bvecfree((struct berval**)&(sop->p.p_bind.bind_ret_saslcreds));
  1060. sop->p.p_bind.bind_creds = NULL;
  1061. sop->p.p_bind.bind_ret_saslcreds = NULL;
  1062. break;
  1063. case SLAPI_OPERATION_COMPARE:
  1064. ava_done((struct ava *)&(sop->p.p_compare.compare_ava));
  1065. break;
  1066. case SLAPI_OPERATION_SEARCH:
  1067. slapi_ch_free((void **)&(sop->p.p_search.search_strfilter));
  1068. charray_free(sop->p.p_search.search_attrs);
  1069. slapi_filter_free(sop->p.p_search.search_filter,1);
  1070. break;
  1071. case SLAPI_OPERATION_MODRDN:
  1072. sop->p.p_modrdn.modrdn_deloldrdn = 0;
  1073. break;
  1074. case SLAPI_OPERATION_EXTENDED:
  1075. slapi_ch_free((void **)&(sop->p.p_extended.exop_oid));
  1076. if (sop->p.p_extended.exop_value)
  1077. ber_bvecfree((struct berval**)&(sop->p.p_extended.exop_value));
  1078. sop->p.p_extended.exop_value = NULL;
  1079. break;
  1080. default:
  1081. break;
  1082. }
  1083. }
  1084. operation_parameters_done(sop);
  1085. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= w_cl5_operation_parameters_done\n" );
  1086. }
  1087. /*
  1088. * Send a set of updates to the replica. Assumes that (1) the replica
  1089. * has already been acquired, (2) that the consumer's update vector has
  1090. * been checked and (3) that it's ok to send incremental updates.
  1091. * Returns:
  1092. * UPDATE_NO_MORE_UPDATES - all updates were sent succussfully
  1093. * UPDATE_TRANSIENT_ERROR - some non-permanent error occurred. Try again later.
  1094. * UPDATE_FATAL_ERROR - some bad, permanent error occurred.
  1095. * UPDATE_SCHEDULE_WINDOW_CLOSED - the schedule window closed on us.
  1096. */
  1097. static int
  1098. send_updates(Private_Repl_Protocol *prp, RUV *remote_update_vector, PRUint32 *num_changes_sent, int do_send)
  1099. {
  1100. CL5Entry entry;
  1101. slapi_operation_parameters op;
  1102. int return_value = UPDATE_NO_MORE_UPDATES;
  1103. int rc;
  1104. int set_mincsn = 0;
  1105. CL5ReplayIterator *changelog_iterator = NULL;
  1106. RUV *current_ruv = ruv_dup(remote_update_vector);
  1107. CSN *mincsn = NULL;
  1108. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> send_updates\n" );
  1109. *num_changes_sent = 0;
  1110. /* Check if the min csn is set in our RUV to see if we need to set it below. */
  1111. ruv_get_min_csn(current_ruv, &mincsn);
  1112. if (!mincsn) {
  1113. set_mincsn = 1;
  1114. } else {
  1115. csn_free(&mincsn);
  1116. }
  1117. /*
  1118. * Iterate over the changelog. Retrieve each update,
  1119. * construct an appropriate LDAP operation,
  1120. * attaching the CSN, and send the change.
  1121. */
  1122. rc = cl5CreateReplayIteratorEx( prp, remote_update_vector, &changelog_iterator, agmt_get_consumerRID(prp->agmt));
  1123. if (CL5_SUCCESS != rc)
  1124. {
  1125. switch (rc)
  1126. {
  1127. case CL5_BAD_DATA: /* invalid parameter passed to the function */
  1128. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1129. "%s: Invalid parameter passed to cl5CreateReplayIterator\n",
  1130. agmt_get_long_name(prp->agmt));
  1131. return_value = UPDATE_FATAL_ERROR;
  1132. break;
  1133. case CL5_BAD_FORMAT: /* db data has unexpected format */
  1134. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1135. "%s: Unexpected format encountered in changelog database\n",
  1136. agmt_get_long_name(prp->agmt));
  1137. return_value = UPDATE_FATAL_ERROR;
  1138. break;
  1139. case CL5_BAD_STATE: /* changelog is in an incorrect state for attempted operation */
  1140. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1141. "%s: Changelog database was in an incorrect state\n",
  1142. agmt_get_long_name(prp->agmt));
  1143. return_value = UPDATE_FATAL_ERROR;
  1144. break;
  1145. case CL5_BAD_DBVERSION: /* changelog has invalid dbversion */
  1146. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1147. "%s: Incorrect dbversion found in changelog database\n",
  1148. agmt_get_long_name(prp->agmt));
  1149. return_value = UPDATE_FATAL_ERROR;
  1150. break;
  1151. case CL5_DB_ERROR: /* database error */
  1152. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1153. "%s: A changelog database error was encountered\n",
  1154. agmt_get_long_name(prp->agmt));
  1155. return_value = UPDATE_FATAL_ERROR;
  1156. break;
  1157. case CL5_NOTFOUND: /* we have no changes to send */
  1158. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1159. "%s: No changes to send\n",
  1160. agmt_get_long_name(prp->agmt));
  1161. return_value = UPDATE_NO_MORE_UPDATES;
  1162. break;
  1163. case CL5_MEMORY_ERROR: /* memory allocation failed */
  1164. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1165. "%s: Memory allocation error occurred\n",
  1166. agmt_get_long_name(prp->agmt));
  1167. return_value = UPDATE_FATAL_ERROR;
  1168. break;
  1169. case CL5_SYSTEM_ERROR: /* NSPR error occurred: use PR_GetError for furhter info */
  1170. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1171. "%s: An NSPR error (%d) occurred\n",
  1172. agmt_get_long_name(prp->agmt), PR_GetError());
  1173. return_value = UPDATE_TRANSIENT_ERROR;
  1174. break;
  1175. case CL5_CSN_ERROR: /* CSN API failed */
  1176. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1177. "%s: A CSN API failure was encountered\n",
  1178. agmt_get_long_name(prp->agmt));
  1179. return_value = UPDATE_TRANSIENT_ERROR;
  1180. break;
  1181. case CL5_RUV_ERROR: /* RUV API failed */
  1182. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1183. "%s: An RUV API failure occurred\n",
  1184. agmt_get_long_name(prp->agmt));
  1185. return_value = UPDATE_TRANSIENT_ERROR;
  1186. break;
  1187. case CL5_OBJSET_ERROR: /* namedobjset api failed */
  1188. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1189. "%s: A namedobject API failure occurred\n",
  1190. agmt_get_long_name(prp->agmt));
  1191. return_value = UPDATE_TRANSIENT_ERROR;
  1192. break;
  1193. case CL5_PURGED_DATA: /* requested data has been purged */
  1194. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1195. "%s: Data required to update replica has been purged. "
  1196. "The replica must be reinitialized.\n",
  1197. agmt_get_long_name(prp->agmt));
  1198. return_value = UPDATE_FATAL_ERROR;
  1199. break;
  1200. case CL5_MISSING_DATA: /* data should be in the changelog, but is missing */
  1201. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1202. "%s: Missing data encountered\n",
  1203. agmt_get_long_name(prp->agmt));
  1204. return_value = UPDATE_FATAL_ERROR;
  1205. break;
  1206. case CL5_UNKNOWN_ERROR: /* unclassified error */
  1207. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1208. "%s: An unknown error was ecountered\n",
  1209. agmt_get_long_name(prp->agmt));
  1210. return_value = UPDATE_TRANSIENT_ERROR;
  1211. break;
  1212. default:
  1213. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1214. "%s: An unknown error (%d) occurred "
  1215. "(cl5CreateReplayIterator)\n",
  1216. agmt_get_long_name(prp->agmt), rc);
  1217. return_value = UPDATE_TRANSIENT_ERROR;
  1218. }
  1219. }
  1220. else
  1221. {
  1222. int finished = 0;
  1223. ConnResult replay_crc;
  1224. char csn_str[CSN_STRSIZE];
  1225. memset ( (void*)&op, 0, sizeof (op) );
  1226. entry.op = &op;
  1227. do {
  1228. int mark_record_done = 0;
  1229. w_cl5_operation_parameters_done ( entry.op );
  1230. memset ( (void*)entry.op, 0, sizeof (op) );
  1231. rc = cl5GetNextOperationToReplay(changelog_iterator, &entry);
  1232. switch (rc)
  1233. {
  1234. case CL5_SUCCESS:
  1235. /* check that we don't return dummy entries */
  1236. if (is_dummy_operation (entry.op))
  1237. {
  1238. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1239. "%s: changelog iteration code returned a dummy entry with csn %s, "
  1240. "skipping ...\n",
  1241. agmt_get_long_name(prp->agmt), csn_as_string(entry.op->csn, PR_FALSE, csn_str));
  1242. continue;
  1243. }
  1244. /* This is where the work actually happens: */
  1245. if (do_send) {
  1246. replay_crc = windows_replay_update(prp, entry.op);
  1247. } else {
  1248. /* Skip sending the updates and just set a success code. */
  1249. replay_crc = CONN_OPERATION_SUCCESS;
  1250. }
  1251. if (CONN_OPERATION_SUCCESS != replay_crc)
  1252. {
  1253. int operation, error;
  1254. windows_conn_get_error(prp->conn, &operation, &error);
  1255. csn_as_string(entry.op->csn, PR_FALSE, csn_str);
  1256. /* Figure out what to do next */
  1257. if (CONN_OPERATION_FAILED == replay_crc)
  1258. {
  1259. /* Map ldap error code to return value */
  1260. if (!windows_ignore_error_and_keep_going(error))
  1261. {
  1262. return_value = UPDATE_TRANSIENT_ERROR;
  1263. finished = 1;
  1264. }
  1265. else
  1266. {
  1267. agmt_inc_last_update_changecount (prp->agmt, csn_get_replicaid(entry.op->csn), 1 /*skipped*/);
  1268. mark_record_done = 1;
  1269. }
  1270. slapi_log_error(finished ? SLAPI_LOG_FATAL : slapi_log_urp, windows_repl_plugin_name,
  1271. "%s: Consumer failed to replay change (uniqueid %s, CSN %s): %s. %s.\n",
  1272. agmt_get_long_name(prp->agmt),
  1273. entry.op->target_address.uniqueid, csn_str,
  1274. ldap_err2string(error),
  1275. finished ? "Will retry later" : "Skipping");
  1276. }
  1277. else if (CONN_NOT_CONNECTED == replay_crc)
  1278. {
  1279. /* We lost the connection - enter backoff state */
  1280. return_value = UPDATE_TRANSIENT_ERROR;
  1281. finished = 1;
  1282. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1283. "%s: Consumer failed to replay change (uniqueid %s, CSN %s): "
  1284. "%s. Will retry later.\n",
  1285. agmt_get_long_name(prp->agmt),
  1286. entry.op->target_address.uniqueid, csn_str,
  1287. error ? ldap_err2string(error) : "Connection lost");
  1288. }
  1289. else if (CONN_TIMEOUT == replay_crc)
  1290. {
  1291. return_value = UPDATE_TIMEOUT;
  1292. finished = 1;
  1293. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1294. "%s: Consumer timed out to replay change (uniqueid %s, CSN %s): "
  1295. "%s.\n",
  1296. agmt_get_long_name(prp->agmt),
  1297. entry.op->target_address.uniqueid, csn_str,
  1298. error ? ldap_err2string(error) : "Timeout");
  1299. }
  1300. else if (CONN_LOCAL_ERROR == replay_crc)
  1301. {
  1302. /*
  1303. * Something bad happened on the local server - enter
  1304. * backoff state.
  1305. */
  1306. return_value = UPDATE_TRANSIENT_ERROR;
  1307. finished = 1;
  1308. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1309. "%s: Failed to replay change (uniqueid %s, CSN %s): "
  1310. "Local error. Will retry later.\n",
  1311. agmt_get_long_name(prp->agmt),
  1312. entry.op->target_address.uniqueid, csn_str);
  1313. }
  1314. }
  1315. else
  1316. {
  1317. /* Positive response received */
  1318. (*num_changes_sent)++;
  1319. agmt_inc_last_update_changecount (prp->agmt, csn_get_replicaid(entry.op->csn), 0 /*replayed*/);
  1320. mark_record_done = 1;
  1321. }
  1322. if (mark_record_done)
  1323. {
  1324. /* If this is the very first change being sent,
  1325. * it's possible that we haven't set a min csn
  1326. * in the RUV yet. This is possible because we
  1327. * simply copy the supplier RUV during the total
  1328. * update process. The supplier RUV will not have
  1329. * a min or max csn set if no changes have ever
  1330. * been written to it's changelog. We need to set
  1331. * the min csn for the consumer here to prevent
  1332. * problems with further sync operations. */
  1333. if (set_mincsn) {
  1334. ruv_set_min_csn(current_ruv, entry.op->csn, NULL);
  1335. set_mincsn = 0;
  1336. }
  1337. /* Bring the consumers (AD) RUV up to date.
  1338. * This sets the max csn. */
  1339. ruv_force_csn_update(current_ruv, entry.op->csn);
  1340. }
  1341. break;
  1342. case CL5_BAD_DATA:
  1343. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1344. "%s: Invalid parameter passed to cl5GetNextOperationToReplay\n",
  1345. agmt_get_long_name(prp->agmt));
  1346. return_value = UPDATE_FATAL_ERROR;
  1347. finished = 1;
  1348. break;
  1349. case CL5_NOTFOUND:
  1350. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1351. "%s: No more updates to send (cl5GetNextOperationToReplay)\n",
  1352. agmt_get_long_name(prp->agmt));
  1353. return_value = UPDATE_NO_MORE_UPDATES;
  1354. finished = 1;
  1355. break;
  1356. case CL5_DB_ERROR:
  1357. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1358. "%s: A database error occurred (cl5GetNextOperationToReplay)\n",
  1359. agmt_get_long_name(prp->agmt));
  1360. return_value = UPDATE_FATAL_ERROR;
  1361. finished = 1;
  1362. break;
  1363. case CL5_BAD_FORMAT:
  1364. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1365. "%s: A malformed changelog entry was encountered (cl5GetNextOperationToReplay)\n",
  1366. agmt_get_long_name(prp->agmt));
  1367. break;
  1368. case CL5_MEMORY_ERROR:
  1369. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1370. "%s: A memory allocation error occurred (cl5GetNextOperationToRepla)\n",
  1371. agmt_get_long_name(prp->agmt));
  1372. return_value = UPDATE_FATAL_ERROR;
  1373. break;
  1374. default:
  1375. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1376. "%s: Unknown error code (%d) returned from cl5GetNextOperationToReplay\n",
  1377. agmt_get_long_name(prp->agmt), rc);
  1378. return_value = UPDATE_TRANSIENT_ERROR;
  1379. break;
  1380. }
  1381. /* Check for protocol shutdown */
  1382. if (prp->terminate)
  1383. {
  1384. return_value = UPDATE_NO_MORE_UPDATES;
  1385. finished = 1;
  1386. }
  1387. if (*num_changes_sent >= MAX_CHANGES_PER_SESSION)
  1388. {
  1389. return_value = UPDATE_YIELD;
  1390. finished = 1;
  1391. }
  1392. } while (!finished);
  1393. w_cl5_operation_parameters_done ( entry.op );
  1394. cl5DestroyReplayIterator(&changelog_iterator);
  1395. }
  1396. /* Save the RUV that we successfully replayed, this ensures that next time we start off at the next changelog record */
  1397. if (current_ruv)
  1398. {
  1399. agmt_set_consumer_ruv(prp->agmt,current_ruv);
  1400. ruv_destroy(&current_ruv);
  1401. }
  1402. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= send_updates\n" );
  1403. return return_value;
  1404. }
  1405. /*
  1406. * XXXggood this should probably be in the superclass, since the full update
  1407. * protocol is going to need it too.
  1408. */
  1409. static int
  1410. windows_inc_stop(Private_Repl_Protocol *prp)
  1411. {
  1412. int return_value;
  1413. PRIntervalTime start, maxwait, now;
  1414. int seconds = 1200;
  1415. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_stop\n" );
  1416. maxwait = PR_SecondsToInterval(seconds);
  1417. prp->terminate = 1;
  1418. event_notify(prp, EVENT_PROTOCOL_SHUTDOWN);
  1419. start = PR_IntervalNow();
  1420. now = start;
  1421. while (!prp->stopped && ((now - start) < maxwait))
  1422. {
  1423. DS_Sleep(PR_SecondsToInterval(1));
  1424. now = PR_IntervalNow();
  1425. }
  1426. if (!prp->stopped)
  1427. {
  1428. /* Isn't listening. Do something drastic. */
  1429. return_value = -1;
  1430. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1431. "%s: windows_inc_stop: protocol does not stop after %d seconds\n",
  1432. agmt_get_long_name(prp->agmt), seconds);
  1433. }
  1434. else
  1435. {
  1436. return_value = 0;
  1437. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1438. "%s: windows_inc_stop: protocol stopped after %d seconds\n",
  1439. agmt_get_long_name(prp->agmt),
  1440. PR_IntervalToSeconds(now-start));
  1441. }
  1442. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_stop\n" );
  1443. return return_value;
  1444. }
  1445. static int
  1446. windows_inc_status(Private_Repl_Protocol *prp)
  1447. {
  1448. int return_value = 0;
  1449. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_status\n" );
  1450. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_status\n" );
  1451. return return_value;
  1452. }
  1453. static void
  1454. windows_inc_notify_update(Private_Repl_Protocol *prp)
  1455. {
  1456. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_notify_update\n" );
  1457. event_notify(prp, EVENT_TRIGGERING_CRITERIA_MET);
  1458. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_notify_update\n" );
  1459. }
  1460. static void
  1461. windows_inc_update_now(Private_Repl_Protocol *prp)
  1462. {
  1463. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_update_now\n" );
  1464. event_notify(prp, EVENT_REPLICATE_NOW);
  1465. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_update_now\n" );
  1466. }
  1467. static void
  1468. windows_inc_notify_agmt_changed(Private_Repl_Protocol *prp)
  1469. {
  1470. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_notify_agmt_changed\n" );
  1471. event_notify(prp, EVENT_AGMT_CHANGED);
  1472. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_notify_agmt_changed\n" );
  1473. }
  1474. static void
  1475. windows_inc_notify_window_opened (Private_Repl_Protocol *prp)
  1476. {
  1477. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_notify_window_opened\n" );
  1478. event_notify(prp, EVENT_WINDOW_OPENED);
  1479. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_notify_window_opened\n" );
  1480. }
  1481. static void
  1482. windows_inc_notify_window_closed (Private_Repl_Protocol *prp)
  1483. {
  1484. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_notify_window_closed\n" );
  1485. event_notify(prp, EVENT_WINDOW_CLOSED);
  1486. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_notify_window_closed\n" );
  1487. }
  1488. Private_Repl_Protocol *
  1489. Windows_Inc_Protocol_new(Repl_Protocol *rp)
  1490. {
  1491. windows_inc_private *rip = NULL;
  1492. Private_Repl_Protocol *prp = (Private_Repl_Protocol *)slapi_ch_calloc(1, sizeof(Private_Repl_Protocol));
  1493. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> Windows_Inc_Protocol_new\n" );
  1494. prp->delete = windows_inc_delete;
  1495. prp->run = windows_inc_run;
  1496. prp->stop = windows_inc_stop;
  1497. prp->status = windows_inc_status;
  1498. prp->notify_update = windows_inc_notify_update;
  1499. prp->notify_agmt_changed = windows_inc_notify_agmt_changed;
  1500. prp->notify_window_opened = windows_inc_notify_window_opened;
  1501. prp->notify_window_closed = windows_inc_notify_window_closed;
  1502. prp->update_now = windows_inc_update_now;
  1503. prp->replica_object = prot_get_replica_object(rp);
  1504. if ((prp->lock = PR_NewLock()) == NULL)
  1505. {
  1506. goto loser;
  1507. }
  1508. if ((prp->cvar = PR_NewCondVar(prp->lock)) == NULL)
  1509. {
  1510. goto loser;
  1511. }
  1512. prp->stopped = 0;
  1513. prp->terminate = 0;
  1514. prp->eventbits = 0;
  1515. prp->conn = prot_get_connection(rp);
  1516. prp->agmt = prot_get_agreement(rp);
  1517. prp->last_acquire_response_code = NSDS50_REPL_REPLICA_READY;
  1518. rip = (void *)slapi_ch_malloc(sizeof(windows_inc_private));
  1519. rip->ruv = NULL;
  1520. rip->backoff = NULL;
  1521. rip->rp = rp;
  1522. prp->private = (void *)rip;
  1523. prp->replica_acquired = PR_FALSE;
  1524. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= Windows_Inc_Protocol_new\n" );
  1525. return prp;
  1526. loser:
  1527. windows_inc_delete(&prp);
  1528. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= Windows_Inc_Protocol_new (loser)\n" );
  1529. return NULL;
  1530. }
  1531. static void
  1532. windows_inc_backoff_expired(time_t timer_fire_time, void *arg)
  1533. {
  1534. Private_Repl_Protocol *prp = (Private_Repl_Protocol *)arg;
  1535. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_backoff_expired\n" );
  1536. PR_ASSERT(NULL != prp);
  1537. event_notify(prp, EVENT_BACKOFF_EXPIRED);
  1538. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_backoff_expired\n" );
  1539. }
  1540. /*
  1541. * Examine the update vector and determine our course of action.
  1542. * There are 3 different possibilities, plus a catch-all error:
  1543. * 1 - no update vector (ruv is NULL). The consumer's replica is
  1544. * pristine, so it needs to be initialized. Return
  1545. * EXAMINE_RUV_PRISTINE_REPLICA.
  1546. * 2 - ruv is present, but its database generation ID doesn't
  1547. * match the local generation ID. This means that either
  1548. * the local replica must be reinitialized from the remote
  1549. * replica or vice-versa. Return
  1550. * EXAMINE_RUV_GENERATION_MISMATCH.
  1551. * 3 - ruv is present, and we have all updates needed to bring
  1552. * the replica up to date using the incremental protocol.
  1553. * return EXAMINE_RUV_OK.
  1554. * 4 - parameter error. Return EXAMINE_RUV_PARAM_ERROR
  1555. */
  1556. static int
  1557. windows_examine_update_vector(Private_Repl_Protocol *prp, RUV *remote_ruv)
  1558. {
  1559. int return_value;
  1560. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_examine_update_vector\n" );
  1561. PR_ASSERT(NULL != prp);
  1562. if (NULL == prp)
  1563. {
  1564. return_value = EXAMINE_RUV_PARAM_ERROR;
  1565. }
  1566. else if (NULL == remote_ruv)
  1567. {
  1568. return_value = EXAMINE_RUV_PRISTINE_REPLICA;
  1569. }
  1570. else
  1571. {
  1572. char *local_gen = NULL;
  1573. char *remote_gen = ruv_get_replica_generation(remote_ruv);
  1574. Object *local_ruv_obj;
  1575. RUV *local_ruv;
  1576. Replica *replica;
  1577. PR_ASSERT(NULL != prp->replica_object);
  1578. replica = object_get_data(prp->replica_object);
  1579. PR_ASSERT(NULL != replica);
  1580. local_ruv_obj = replica_get_ruv (replica);
  1581. if (NULL != local_ruv_obj)
  1582. {
  1583. local_ruv = (RUV*) object_get_data (local_ruv_obj);
  1584. PR_ASSERT (local_ruv);
  1585. local_gen = ruv_get_replica_generation(local_ruv);
  1586. object_release (local_ruv_obj);
  1587. }
  1588. if (NULL == remote_gen || NULL == local_gen || strcmp(remote_gen, local_gen) != 0)
  1589. {
  1590. return_value = EXAMINE_RUV_GENERATION_MISMATCH;
  1591. }
  1592. else
  1593. {
  1594. return_value = EXAMINE_RUV_OK;
  1595. }
  1596. slapi_ch_free((void**)&remote_gen);
  1597. slapi_ch_free((void**)&local_gen);
  1598. }
  1599. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_examine_update_vector\n" );
  1600. return return_value;
  1601. }
  1602. /* this function converts an aquisition code to a string - for debug output */
  1603. static const char*
  1604. acquire2name (int code)
  1605. {
  1606. switch (code)
  1607. {
  1608. case ACQUIRE_SUCCESS: return "success";
  1609. case ACQUIRE_REPLICA_BUSY: return "replica_busy";
  1610. case ACQUIRE_FATAL_ERROR: return "fatal_error";
  1611. case ACQUIRE_CONSUMER_WAS_UPTODATE: return "consumer_was_uptodate";
  1612. case ACQUIRE_TRANSIENT_ERROR: return "transient_error";
  1613. default: return "invalid_code";
  1614. }
  1615. }
  1616. /* this function converts a state to its name - for debug output */
  1617. static const char*
  1618. state2name (int state)
  1619. {
  1620. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> state2name\n" );
  1621. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= state2name\n" );
  1622. switch (state)
  1623. {
  1624. case STATE_START: return "start";
  1625. case STATE_WAIT_WINDOW_OPEN: return "wait_for_window_to_open";
  1626. case STATE_WAIT_CHANGES: return "wait_for_changes";
  1627. case STATE_READY_TO_ACQUIRE: return "ready_to_acquire_replica";
  1628. case STATE_BACKOFF_START: return "start_backoff";
  1629. case STATE_BACKOFF: return "backoff";
  1630. case STATE_SENDING_UPDATES: return "sending_updates";
  1631. case STATE_STOP_FATAL_ERROR: return "stop_fatal_error";
  1632. case STATE_STOP_FATAL_ERROR_PART2: return "stop_fatal_error";
  1633. case STATE_STOP_NORMAL_TERMINATION: return "stop_normal_termination";
  1634. default: return "invalid_state";
  1635. }
  1636. }
  1637. /* this function convert s an event to its name - for debug output */
  1638. static const char*
  1639. event2name (int event)
  1640. {
  1641. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> event2name\n" );
  1642. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= event2name\n" );
  1643. switch (event)
  1644. {
  1645. case EVENT_WINDOW_OPENED: return "update_window_opened";
  1646. case EVENT_WINDOW_CLOSED: return "update_window_closed";
  1647. case EVENT_TRIGGERING_CRITERIA_MET: return "data_modified";
  1648. case EVENT_BACKOFF_EXPIRED: return "backoff_timer_expired";
  1649. case EVENT_REPLICATE_NOW: return "replicate_now";
  1650. case EVENT_PROTOCOL_SHUTDOWN: return "protocol_shutdown";
  1651. case EVENT_AGMT_CHANGED: return "agreement_changed";
  1652. case EVENT_RUN_DIRSYNC: return "run_dirsync";
  1653. default: return "invalid_event";
  1654. }
  1655. }
  1656. static void
  1657. periodic_dirsync(time_t when, void *arg)
  1658. {
  1659. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> periodic_dirsync\n" );
  1660. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1661. "Running Dirsync \n");
  1662. event_notify( (Private_Repl_Protocol*) arg, EVENT_RUN_DIRSYNC);
  1663. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= periodic_dirsync\n" );
  1664. }