windows_inc_protocol.c 63 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814
  1. /** BEGIN COPYRIGHT BLOCK
  2. * This Program is free software; you can redistribute it and/or modify it under
  3. * the terms of the GNU General Public License as published by the Free Software
  4. * Foundation; version 2 of the License.
  5. *
  6. * This Program is distributed in the hope that it will be useful, but WITHOUT
  7. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  8. * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  9. *
  10. * You should have received a copy of the GNU General Public License along with
  11. * this Program; if not, write to the Free Software Foundation, Inc., 59 Temple
  12. * Place, Suite 330, Boston, MA 02111-1307 USA.
  13. *
  14. * In addition, as a special exception, Red Hat, Inc. gives You the additional
  15. * right to link the code of this Program with code not covered under the GNU
  16. * General Public License ("Non-GPL Code") and to distribute linked combinations
  17. * including the two, subject to the limitations in this paragraph. Non-GPL Code
  18. * permitted under this exception must only link to the code of this Program
  19. * through those well defined interfaces identified in the file named EXCEPTION
  20. * found in the source code files (the "Approved Interfaces"). The files of
  21. * Non-GPL Code may instantiate templates or use macros or inline functions from
  22. * the Approved Interfaces without causing the resulting work to be covered by
  23. * the GNU General Public License. Only Red Hat, Inc. may make changes or
  24. * additions to the list of Approved Interfaces. You must obey the GNU General
  25. * Public License in all respects for all of the Program code and other code used
  26. * in conjunction with the Program except the Non-GPL Code covered by this
  27. * exception. If you modify this file, you may extend this exception to your
  28. * version of the file, but you are not obligated to do so. If you do not wish to
  29. * provide this exception without modification, you must delete this exception
  30. * statement from your version and license this file solely under the GPL without
  31. * exception.
  32. *
  33. *
  34. * Copyright (C) 2001 Sun Microsystems, Inc. Used by permission.
  35. * Copyright (C) 2005 Red Hat, Inc.
  36. * All rights reserved.
  37. * END COPYRIGHT BLOCK **/
  38. #ifdef HAVE_CONFIG_H
  39. # include <config.h>
  40. #endif
  41. /* repl5_inc_protocol.c */
  42. /*
  43. The Prot_Incremental object implements the DS 5.0 multi-master incremental
  44. replication protocol.
  45. Stuff to do:
  46. - Need to figure out how asynchronous events end up in here. They are:
  47. - entry updated in replicated area.
  48. - backoff timeout
  49. - enter/leave.
  50. Perhaps these events should be properties of the main protocol.
  51. */
  52. #include "repl.h"
  53. #include "repl5.h"
  54. #include "windowsrepl.h"
  55. #include "windows_prot_private.h"
  56. #include "slap.h" /* PSEUDO_ATTR_UNHASHED */
  57. #include "repl5_ruv.h"
  58. #include "cl5_api.h"
  59. #include "slapi-plugin.h"
  60. extern int slapi_log_urp;
  61. /*** from proto-slap.h ***/
  62. void ava_done(struct ava *ava);
  63. typedef struct windows_inc_private
  64. {
  65. char *ruv; /* RUV on remote replica (use diff type for this? - ggood */
  66. Backoff_Timer *backoff;
  67. Repl_Protocol *rp;
  68. PRLock *lock;
  69. PRUint32 eventbits;
  70. } windows_inc_private;
  71. /* Various states the incremental protocol can pass through */
  72. #define STATE_START 0 /* ONREPL - should we rename this - we don't use it just to start up? */
  73. #define STATE_WAIT_WINDOW_OPEN 1
  74. #define STATE_WAIT_CHANGES 2
  75. #define STATE_READY_TO_ACQUIRE 3
  76. #define STATE_BACKOFF_START 4 /* ONREPL - can we combine BACKOFF_START and BACKOFF states? */
  77. #define STATE_BACKOFF 5
  78. #define STATE_SENDING_UPDATES 6
  79. #define STATE_STOP_FATAL_ERROR 7
  80. #define STATE_STOP_FATAL_ERROR_PART2 8
  81. #define STATE_STOP_NORMAL_TERMINATION 9
  82. /* Events (synchronous and asynchronous; these are bits) */
  83. #define EVENT_WINDOW_OPENED 1
  84. #define EVENT_WINDOW_CLOSED 2
  85. #define EVENT_TRIGGERING_CRITERIA_MET 4 /* ONREPL - should we rename this to EVENT_CHANGE_AVAILABLE */
  86. #define EVENT_BACKOFF_EXPIRED 8
  87. #define EVENT_REPLICATE_NOW 16
  88. #define EVENT_PROTOCOL_SHUTDOWN 32
  89. #define EVENT_AGMT_CHANGED 64
  90. #define EVENT_RUN_DIRSYNC 128
  91. #define UPDATE_NO_MORE_UPDATES 201
  92. #define UPDATE_TRANSIENT_ERROR 202
  93. #define UPDATE_FATAL_ERROR 203
  94. #define UPDATE_SCHEDULE_WINDOW_CLOSED 204
  95. #define UPDATE_CONNECTION_LOST 205
  96. #define UPDATE_TIMEOUT 206
  97. #define UPDATE_YIELD 207
  98. /* Return codes from examine_update_vector */
  99. #define EXAMINE_RUV_PRISTINE_REPLICA 401
  100. #define EXAMINE_RUV_GENERATION_MISMATCH 402
  101. #define EXAMINE_RUV_REPLICA_TOO_OLD 403
  102. #define EXAMINE_RUV_OK 404
  103. #define EXAMINE_RUV_PARAM_ERROR 405
  104. #define MAX_CHANGES_PER_SESSION 10000
  105. /*
  106. * Maximum time to wait between replication sessions. If we
  107. * don't see any updates for a period equal to this interval,
  108. * we go ahead and start a replication session, just to be safe
  109. */
  110. #define MAX_WAIT_BETWEEN_SESSIONS PR_SecondsToInterval(60 * 5) /* 5 minutes */
  111. /*
  112. * tests if the protocol has been shutdown and we need to quit
  113. * event_occurred resets the bits in the bit flag, so whoever tests for shutdown
  114. * resets the flags, so the next one who tests for shutdown won't get it, so we
  115. * also look at the terminate flag
  116. */
  117. #define PROTOCOL_IS_SHUTDOWN(prp) (event_occurred(prp, EVENT_PROTOCOL_SHUTDOWN) || prp->terminate)
  118. /* Forward declarations */
  119. static PRUint32 event_occurred(Private_Repl_Protocol *prp, PRUint32 event);
  120. static void reset_events (Private_Repl_Protocol *prp);
  121. static void protocol_sleep(Private_Repl_Protocol *prp, PRIntervalTime duration);
  122. static int send_updates(Private_Repl_Protocol *prp, RUV *ruv, PRUint32 *num_changes_sent, int do_send);
  123. static void windows_inc_backoff_expired(time_t timer_fire_time, void *arg);
  124. static int windows_examine_update_vector(Private_Repl_Protocol *prp, RUV *ruv);
  125. static const char* state2name (int state);
  126. static const char* event2name (int event);
  127. static const char* acquire2name (int code);
  128. static void periodic_dirsync(time_t when, void *arg);
  129. static Slapi_Eq_Context dirsync = NULL;
  130. /*
  131. * It's specifically ok to delete a protocol instance that
  132. * is currently running. The instance will be shut down, and
  133. * then resources will be freed. Since a graceful shutdown is
  134. * attempted, this function may take some time to complete.
  135. */
  136. static void
  137. windows_inc_delete(Private_Repl_Protocol **prpp)
  138. {
  139. int rc;
  140. windows_inc_private *prp_priv = (windows_inc_private *)(*prpp)->private;
  141. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_delete\n" );
  142. /* First, stop the protocol if it isn't already stopped */
  143. /* Then, delete all resources used by the protocol */
  144. rc = slapi_eq_cancel(dirsync);
  145. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  146. "windows_inc_delete: dirsync: %p, rval: %d\n", dirsync, rc);
  147. /* if backoff is set, delete it (from EQ, as well) */
  148. if (prp_priv->backoff) {
  149. backoff_delete(&prp_priv->backoff);
  150. }
  151. if (!(*prpp)->stopped) {
  152. (*prpp)->stopped = 1;
  153. (*prpp)->stop(*prpp);
  154. }
  155. if ((*prpp)->lock) {
  156. PR_DestroyLock((*prpp)->lock);
  157. (*prpp)->lock = NULL;
  158. }
  159. if ((*prpp)->cvar) {
  160. PR_DestroyCondVar((*prpp)->cvar);
  161. (*prpp)->cvar = NULL;
  162. }
  163. slapi_ch_free((void **)&(*prpp)->private);
  164. slapi_ch_free((void **)prpp);
  165. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_delete\n" );
  166. }
  167. /* helper function */
  168. void
  169. w_set_pause_and_busy_time(Private_Repl_Protocol *prp, long *pausetime, long *busywaittime)
  170. {
  171. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> w_set_pause_and_busy_time\n" );
  172. /* If neither are set, set busy time to its default */
  173. if (!*pausetime && !*busywaittime)
  174. {
  175. *busywaittime = repl5_get_backoff_min(prp);
  176. }
  177. /* pause time must be at least 1 more than the busy backoff time */
  178. if (*pausetime && !*busywaittime)
  179. {
  180. /*
  181. * user specified a pause time but no busy wait time - must
  182. * set busy wait time to 1 less than pause time - if pause
  183. * time is 1, we must set it to 2
  184. */
  185. if (*pausetime < 2)
  186. {
  187. *pausetime = 2;
  188. }
  189. *busywaittime = *pausetime - 1;
  190. }
  191. else if (!*pausetime && *busywaittime)
  192. {
  193. /*
  194. * user specified a busy wait time but no pause time - must
  195. * set pause time to 1 more than busy wait time
  196. */
  197. *pausetime = *busywaittime + 1;
  198. }
  199. else if (*pausetime && *busywaittime && *pausetime <= *busywaittime)
  200. {
  201. /*
  202. * user specified both pause and busy wait times, but the pause
  203. * time was <= busy wait time - pause time must be at least
  204. * 1 more than the busy wait time
  205. */
  206. *pausetime = *busywaittime + 1;
  207. }
  208. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= w_set_pause_and_busy_time\n" );
  209. }
  210. /*
  211. * Do the incremental protocol.
  212. *
  213. * What's going on here? This thing is a state machine. It has the
  214. * following states:
  215. *
  216. * State transition table:
  217. *
  218. * Curr State Condition/Event Next State
  219. * ---------- ------------ -----------
  220. * START schedule window is open ACQUIRE_REPLICA
  221. * schedule window is closed WAIT_WINDOW_OPEN
  222. * WAIT_WINDOW_OPEN schedule change START
  223. * replicate now ACQUIRE_REPLICA
  224. * schedule window opens ACQUIRE_REPLICA
  225. * ACQUIRE_REPLICA acquired replica SEND_CHANGES
  226. * failed to acquire - transient error START_BACKOFF
  227. * failed to acquire - fatal error STOP_FATAL_ERROR
  228. * SEND_CHANGES can't update CONSUMER_NEEDS_REINIT
  229. * no changes to send WAIT_CHANGES
  230. * can't send - thransient error START_BACKOF
  231. * can't send - window closed WAIT_WINDOW_OPEN
  232. * can'r send - fatal error STOP_FATAL_ERROR
  233. * START_BACKOF replicate now ACQUIRE_REPLICA
  234. * schedule changes START
  235. * schedule window closes WAIT_WINDOW_OPEN
  236. * backoff expires & can acquire SEND_CHANGES
  237. * backoff expires & can't acquire-trans BACKOFF
  238. * backoff expires & can't acquire-fatal STOP_FATAL_ERROR
  239. * BACKOF replicate now ACQUIRE_REPLICA
  240. * schedule changes START
  241. * schedule window closes WAIT_WINDOW_OPEN
  242. * backoff expires & can acquire SEND_CHANGES
  243. * backoff expires & can't acquire-trans BACKOFF
  244. * backoff expires & can't acquire-fatal STOP_FATAL_ERROR
  245. * WAIT_CHANGES schedule window closes WAIT_WINDOW_OPEN
  246. * replicate_now ACQUIRE_REPLICA
  247. * change available ACQUIRE_REPLICA
  248. * schedule_change START
  249. */
  250. /*
  251. * DBDB: what follows is quite possibly the worst code I have ever seen.
  252. * Unfortunately we chose not to re-write it when we did the windows sync version.
  253. */
  254. /*
  255. * Main state machine for the incremental protocol. This routine will,
  256. * under normal circumstances, not return until the protocol is shut
  257. * down.
  258. */
  259. static void
  260. windows_inc_run(Private_Repl_Protocol *prp)
  261. {
  262. int current_state = STATE_START;
  263. int next_state = STATE_START;
  264. windows_inc_private *prp_priv = (windows_inc_private *)prp->private;
  265. int done = 0;
  266. int e1 = 0;
  267. RUV *ruv = NULL;
  268. Replica *replica = NULL;
  269. int wait_change_timer_set = 0;
  270. time_t last_start_time = 0;
  271. PRUint32 num_changes_sent = 0;
  272. /* use a different backoff timer strategy for ACQUIRE_REPLICA_BUSY errors */
  273. PRBool use_busy_backoff_timer = PR_FALSE;
  274. long pausetime = 0;
  275. long busywaittime = 0;
  276. unsigned long current_interval = 0;
  277. unsigned long interval = 0;
  278. int one_way;
  279. PRBool run_dirsync = PR_FALSE;
  280. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_run\n" );
  281. prp->stopped = 0;
  282. prp->terminate = 0;
  283. windows_private_load_dirsync_cookie(prp->agmt);
  284. one_way = windows_private_get_one_way(prp->agmt);
  285. do {
  286. int rc = 0;
  287. /* Take action, based on current state, and compute new state. */
  288. switch (current_state)
  289. {
  290. case STATE_START:
  291. dev_debug("windows_inc_run(STATE_START)");
  292. if (PROTOCOL_IS_SHUTDOWN(prp))
  293. {
  294. done = 1;
  295. break;
  296. }
  297. /*
  298. * Our initial state. See if we're in a schedule window. If
  299. * so, then we're ready to acquire the replica and see if it
  300. * needs any updates from us. If not, then wait for the window
  301. * to open.
  302. */
  303. if (agmt_schedule_in_window_now(prp->agmt))
  304. {
  305. next_state = STATE_READY_TO_ACQUIRE;
  306. } else
  307. {
  308. next_state = STATE_WAIT_WINDOW_OPEN;
  309. }
  310. /* we can get here from other states because some events happened and were
  311. not cleared. For instance when we wake up in STATE_WAIT_CHANGES state.
  312. Since this is a fresh start state, we should clear all events */
  313. /* ONREPL - this does not feel right - we should take another look
  314. at this state machine */
  315. reset_events (prp);
  316. /* Cancel any linger timer that might be in effect... */
  317. windows_conn_cancel_linger(prp->conn);
  318. /* ... and disconnect, if currently connected */
  319. windows_conn_disconnect(prp->conn);
  320. /* get the new pause time, if any */
  321. pausetime = agmt_get_pausetime(prp->agmt);
  322. /* get the new busy wait time, if any */
  323. busywaittime = agmt_get_busywaittime(prp->agmt);
  324. if (pausetime || busywaittime)
  325. {
  326. /* helper function to make sure they are set correctly */
  327. w_set_pause_and_busy_time(prp, &pausetime, &busywaittime);
  328. }
  329. /* Check if the interval changed */
  330. interval = windows_private_get_sync_interval(prp->agmt) * 1000;
  331. if(interval != current_interval){
  332. current_interval = interval;
  333. if(dirsync){
  334. int rc = slapi_eq_cancel(dirsync);
  335. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  336. "windows_inc_runs: cancelled dirsync: %p, rval: %d\n",
  337. dirsync, rc);
  338. }
  339. dirsync = slapi_eq_repeat(periodic_dirsync, (void*) prp, (time_t)0 , interval);
  340. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  341. "windows_inc_runs: new dirsync: %p\n", dirsync);
  342. }
  343. break;
  344. case STATE_WAIT_WINDOW_OPEN:
  345. /*
  346. * We're waiting for a schedule window to open. If one did,
  347. * or we receive a "replicate now" event, then start a protocol
  348. * session immediately. If the replication schedule changed, go
  349. * back to start. Otherwise, go back to sleep.
  350. */
  351. dev_debug("windows_inc_run(STATE_WAIT_WINDOW_OPEN)");
  352. if (PROTOCOL_IS_SHUTDOWN(prp))
  353. {
  354. done = 1;
  355. break;
  356. }
  357. else if (event_occurred(prp, EVENT_WINDOW_OPENED))
  358. {
  359. next_state = STATE_READY_TO_ACQUIRE;
  360. }
  361. else if (event_occurred(prp, EVENT_REPLICATE_NOW))
  362. {
  363. next_state = STATE_READY_TO_ACQUIRE;
  364. }
  365. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  366. {
  367. next_state = STATE_START;
  368. run_dirsync = PR_TRUE;
  369. windows_conn_set_agmt_changed(prp->conn);
  370. }
  371. else if (event_occurred(prp, EVENT_TRIGGERING_CRITERIA_MET)) /* change available */
  372. {
  373. /* just ignore it and go to sleep */
  374. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  375. }
  376. else if ((e1 = event_occurred(prp, EVENT_WINDOW_CLOSED)) ||
  377. event_occurred(prp, EVENT_BACKOFF_EXPIRED))
  378. {
  379. /* this events - should not occur - log a warning and go to sleep */
  380. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  381. "%s: Incremental protocol: "
  382. "event %s should not occur in state %s; going to sleep\n",
  383. agmt_get_long_name(prp->agmt),
  384. e1 ? event2name(EVENT_WINDOW_CLOSED) : event2name(EVENT_BACKOFF_EXPIRED),
  385. state2name(current_state));
  386. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  387. }
  388. else if (event_occurred(prp, EVENT_RUN_DIRSYNC)) /* periodic_dirsync */
  389. {
  390. /* just ignore it and go to sleep */
  391. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  392. }
  393. else
  394. {
  395. /* wait until window opens or an event occurs */
  396. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  397. "%s: Incremental protocol: "
  398. "waiting for update window to open\n", agmt_get_long_name(prp->agmt));
  399. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  400. }
  401. break;
  402. case STATE_WAIT_CHANGES:
  403. /*
  404. * We're in a replication window, but we're waiting for more
  405. * changes to accumulate before we actually hook up and send
  406. * them.
  407. */
  408. dev_debug("windows_inc_run(STATE_WAIT_CHANGES)");
  409. if (PROTOCOL_IS_SHUTDOWN(prp))
  410. {
  411. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): PROTOCOL_IS_SHUTING_DOWN -> end windows_inc_run\n");
  412. done = 1;
  413. break;
  414. }
  415. else if (event_occurred(prp, EVENT_REPLICATE_NOW))
  416. {
  417. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_REPLICATE_NOW received -> STATE_READY_TO_ACQUIRE\n");
  418. next_state = STATE_READY_TO_ACQUIRE;
  419. wait_change_timer_set = 0;
  420. /* We also want to run dirsync on a 'replicate now' event */
  421. run_dirsync = PR_TRUE;
  422. }
  423. else if ( event_occurred(prp, EVENT_RUN_DIRSYNC))
  424. {
  425. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_REPLICATE_NOW received -> STATE_READY_TO_ACQUIRE\n");
  426. next_state = STATE_READY_TO_ACQUIRE;
  427. wait_change_timer_set = 0;
  428. run_dirsync = PR_TRUE;
  429. }
  430. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  431. {
  432. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_AGMT_CHANGED received -> STATE_START\n");
  433. next_state = STATE_START;
  434. windows_conn_set_agmt_changed(prp->conn);
  435. wait_change_timer_set = 0;
  436. /* We also want to run dirsync on a 'agreement changed' event, because that's how we receive 'send updates now' */
  437. run_dirsync = PR_TRUE;
  438. }
  439. else if (event_occurred(prp, EVENT_WINDOW_CLOSED))
  440. {
  441. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_WINDOW_CLOSED received -> STATE_WAIT_WINDOW_OPEN\n");
  442. next_state = STATE_WAIT_WINDOW_OPEN;
  443. wait_change_timer_set = 0;
  444. }
  445. else if (event_occurred(prp, EVENT_TRIGGERING_CRITERIA_MET) )
  446. {
  447. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): EVENT_TRIGGERING_CRITERIA_MET received -> STATE_READY_TO_ACQUIRE\n");
  448. next_state = STATE_READY_TO_ACQUIRE;
  449. wait_change_timer_set = 0;
  450. /* We want to run dirsync to catch values generated by AD after
  451. * it process operations that we send to it (such as GUID values). */
  452. run_dirsync = PR_TRUE;
  453. }
  454. else if ((e1 = event_occurred(prp, EVENT_WINDOW_OPENED)) ||
  455. event_occurred(prp, EVENT_BACKOFF_EXPIRED))
  456. {
  457. /* this events - should not occur - log a warning and clear the event */
  458. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name, "%s: Incremental protocol: "
  459. "event %s should not occur in state %s\n",
  460. agmt_get_long_name(prp->agmt),
  461. e1 ? event2name(EVENT_WINDOW_OPENED) : event2name(EVENT_BACKOFF_EXPIRED),
  462. state2name(current_state));
  463. wait_change_timer_set = 0;
  464. }
  465. else
  466. {
  467. if (wait_change_timer_set)
  468. {
  469. /* We are here because our timer expired */
  470. dev_debug("windows_inc_run(STATE_WAIT_CHANGES): wait_change_timer_set expired -> STATE_START\n");
  471. next_state = STATE_START;
  472. run_dirsync = PR_TRUE;
  473. wait_change_timer_set = 0;
  474. }
  475. else
  476. {
  477. /* We are here because the last replication session
  478. * finished or aborted.
  479. */
  480. wait_change_timer_set = 1;
  481. protocol_sleep(prp, MAX_WAIT_BETWEEN_SESSIONS);
  482. }
  483. }
  484. break;
  485. case STATE_READY_TO_ACQUIRE:
  486. dev_debug("windows_inc_run(STATE_READY_TO_ACQUIRE)");
  487. if (PROTOCOL_IS_SHUTDOWN(prp))
  488. {
  489. done = 1;
  490. break;
  491. }
  492. /* ONREPL - at this state we unconditionally acquire the replica
  493. ignoring all events. Not sure if this is good */
  494. object_acquire(prp->replica_object);
  495. rc = windows_acquire_replica(prp, &ruv , (run_dirsync == 0) /* yes, check the consumer RUV for incremental, but not if we're going to dirsync afterwards */);
  496. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  497. "windows_acquire_replica returned %s (%d)\n",
  498. acquire2name(rc),
  499. rc);
  500. use_busy_backoff_timer = PR_FALSE; /* default */
  501. if (rc == ACQUIRE_SUCCESS)
  502. {
  503. next_state = STATE_SENDING_UPDATES;
  504. }
  505. else if (rc == ACQUIRE_REPLICA_BUSY)
  506. {
  507. next_state = STATE_BACKOFF_START;
  508. use_busy_backoff_timer = PR_TRUE;
  509. }
  510. else if (rc == ACQUIRE_CONSUMER_WAS_UPTODATE)
  511. {
  512. next_state = STATE_WAIT_CHANGES;
  513. }
  514. else if (rc == ACQUIRE_TRANSIENT_ERROR)
  515. {
  516. next_state = STATE_BACKOFF_START;
  517. }
  518. else if (rc == ACQUIRE_FATAL_ERROR)
  519. {
  520. next_state = STATE_STOP_FATAL_ERROR;
  521. }
  522. if (rc != ACQUIRE_SUCCESS)
  523. {
  524. int optype, ldaprc;
  525. windows_conn_get_error(prp->conn, &optype, &ldaprc);
  526. agmt_set_last_update_status(prp->agmt, ldaprc,
  527. prp->last_acquire_response_code, NULL);
  528. }
  529. object_release(prp->replica_object);
  530. break;
  531. case STATE_BACKOFF_START:
  532. dev_debug("windows_inc_run(STATE_BACKOFF_START)");
  533. if (PROTOCOL_IS_SHUTDOWN(prp))
  534. {
  535. done = 1;
  536. break;
  537. }
  538. if (event_occurred(prp, EVENT_REPLICATE_NOW) || event_occurred(prp, EVENT_RUN_DIRSYNC))
  539. {
  540. next_state = STATE_READY_TO_ACQUIRE;
  541. }
  542. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  543. {
  544. next_state = STATE_START;
  545. run_dirsync = PR_TRUE; /* Also trigger dirsync for the 'send updates now' feature */
  546. windows_conn_set_agmt_changed(prp->conn);
  547. }
  548. else if (event_occurred (prp, EVENT_WINDOW_CLOSED))
  549. {
  550. next_state = STATE_WAIT_WINDOW_OPEN;
  551. }
  552. else if (event_occurred (prp, EVENT_TRIGGERING_CRITERIA_MET))
  553. {
  554. /* consume and ignore */
  555. }
  556. else if ((e1 = event_occurred (prp, EVENT_WINDOW_OPENED)) ||
  557. event_occurred (prp, EVENT_BACKOFF_EXPIRED))
  558. {
  559. /* This should never happen */
  560. /* this events - should not occur - log a warning and go to sleep */
  561. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  562. "%s: Incremental protocol: event %s should not occur in state %s\n",
  563. agmt_get_long_name(prp->agmt),
  564. e1 ? event2name(EVENT_WINDOW_OPENED) : event2name(EVENT_BACKOFF_EXPIRED),
  565. state2name(current_state));
  566. }
  567. else
  568. {
  569. /* Set up the backoff timer to wake us up at the appropriate time */
  570. /* if previous backoff set up, delete it. */
  571. if (prp_priv->backoff) {
  572. backoff_delete(&prp_priv->backoff);
  573. }
  574. if (use_busy_backoff_timer)
  575. {
  576. /* we received a busy signal from the consumer, wait for a while */
  577. if (!busywaittime){
  578. busywaittime = repl5_get_backoff_min(prp);
  579. }
  580. prp_priv->backoff = backoff_new(BACKOFF_FIXED, busywaittime, busywaittime);
  581. }
  582. else
  583. {
  584. prp_priv->backoff = backoff_new(BACKOFF_EXPONENTIAL, repl5_get_backoff_min(prp),
  585. repl5_get_backoff_max(prp) );
  586. }
  587. next_state = STATE_BACKOFF;
  588. backoff_reset(prp_priv->backoff, windows_inc_backoff_expired, (void *)prp);
  589. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  590. use_busy_backoff_timer = PR_FALSE;
  591. }
  592. break;
  593. case STATE_BACKOFF:
  594. /*
  595. * We're in a backoff state.
  596. */
  597. dev_debug("windows_inc_run(STATE_BACKOFF)");
  598. if (PROTOCOL_IS_SHUTDOWN(prp))
  599. {
  600. if (prp_priv->backoff)
  601. backoff_delete(&prp_priv->backoff);
  602. done = 1;
  603. break;
  604. }
  605. else if (event_occurred(prp, EVENT_REPLICATE_NOW) || event_occurred(prp, EVENT_RUN_DIRSYNC))
  606. {
  607. next_state = STATE_READY_TO_ACQUIRE;
  608. }
  609. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  610. {
  611. next_state = STATE_START;
  612. run_dirsync = PR_TRUE;
  613. windows_conn_set_agmt_changed(prp->conn);
  614. /* Destroy the backoff timer, since we won't need it anymore */
  615. if (prp_priv->backoff)
  616. backoff_delete(&prp_priv->backoff);
  617. }
  618. else if (event_occurred(prp, EVENT_WINDOW_CLOSED))
  619. {
  620. next_state = STATE_WAIT_WINDOW_OPEN;
  621. /* Destroy the backoff timer, since we won't need it anymore */
  622. if (prp_priv->backoff)
  623. backoff_delete(&prp_priv->backoff);
  624. }
  625. else if (event_occurred(prp, EVENT_BACKOFF_EXPIRED))
  626. {
  627. rc = windows_acquire_replica(prp, &ruv, 1 /* check RUV for incremental */);
  628. use_busy_backoff_timer = PR_FALSE;
  629. if (rc == ACQUIRE_SUCCESS)
  630. {
  631. next_state = STATE_SENDING_UPDATES;
  632. }
  633. else if (rc == ACQUIRE_REPLICA_BUSY)
  634. {
  635. next_state = STATE_BACKOFF;
  636. use_busy_backoff_timer = PR_TRUE;
  637. }
  638. else if (rc == ACQUIRE_CONSUMER_WAS_UPTODATE)
  639. {
  640. next_state = STATE_WAIT_CHANGES;
  641. }
  642. else if (rc == ACQUIRE_TRANSIENT_ERROR)
  643. {
  644. next_state = STATE_BACKOFF;
  645. }
  646. else if (rc == ACQUIRE_FATAL_ERROR)
  647. {
  648. next_state = STATE_STOP_FATAL_ERROR;
  649. }
  650. if (rc != ACQUIRE_SUCCESS)
  651. {
  652. int optype, ldaprc;
  653. windows_conn_get_error(prp->conn, &optype, &ldaprc);
  654. agmt_set_last_update_status(prp->agmt, ldaprc,
  655. prp->last_acquire_response_code, NULL);
  656. }
  657. /*
  658. * We either need to step the backoff timer, or
  659. * destroy it if we don't need it anymore.
  660. */
  661. if (STATE_BACKOFF == next_state)
  662. {
  663. time_t next_fire_time;
  664. time_t now;
  665. /* Step the backoff timer */
  666. time(&now);
  667. next_fire_time = backoff_step(prp_priv->backoff);
  668. /* And go back to sleep */
  669. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  670. "%s: Replication session backing off for %ld seconds\n",
  671. agmt_get_long_name(prp->agmt),
  672. next_fire_time - now);
  673. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  674. }
  675. else
  676. {
  677. /* Destroy the backoff timer, since we won't need it anymore */
  678. backoff_delete(&prp_priv->backoff);
  679. }
  680. use_busy_backoff_timer = PR_FALSE;
  681. }
  682. else if (event_occurred(prp, EVENT_TRIGGERING_CRITERIA_MET))
  683. {
  684. /* changes are available */
  685. if ( prp_priv->backoff == NULL || backoff_expired (prp_priv->backoff, 60) )
  686. {
  687. /*
  688. * Have seen cases that the agmt stuck here forever since
  689. * somehow the backoff timer was not in event queue anymore.
  690. * If the backoff timer has expired more than 60 seconds,
  691. * destroy it.
  692. */
  693. if ( prp_priv->backoff )
  694. backoff_delete(&prp_priv->backoff);
  695. next_state = STATE_READY_TO_ACQUIRE;
  696. }
  697. else
  698. {
  699. /* ignore changes and go to sleep */
  700. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  701. }
  702. }
  703. else if (event_occurred(prp, EVENT_WINDOW_OPENED))
  704. {
  705. /* this should never happen - log an error and go to sleep */
  706. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name, "%s: Incremental protocol: "
  707. "event %s should not occur in state %s; going to sleep\n",
  708. agmt_get_long_name(prp->agmt),
  709. event2name(EVENT_WINDOW_OPENED), state2name(current_state));
  710. protocol_sleep(prp, PR_INTERVAL_NO_TIMEOUT);
  711. }
  712. break;
  713. case STATE_SENDING_UPDATES:
  714. dev_debug("windows_inc_run(STATE_SENDING_UPDATES)");
  715. agmt_set_update_in_progress(prp->agmt, PR_TRUE);
  716. num_changes_sent = 0;
  717. last_start_time = current_time();
  718. agmt_set_last_update_start(prp->agmt, last_start_time);
  719. /*
  720. * We've acquired the replica, and are ready to send any
  721. * needed updates.
  722. */
  723. if (PROTOCOL_IS_SHUTDOWN(prp))
  724. {
  725. windows_release_replica (prp);
  726. done = 1;
  727. agmt_set_update_in_progress(prp->agmt, PR_FALSE);
  728. agmt_set_last_update_end(prp->agmt, current_time());
  729. /* MAB: I don't find the following status correct. How do we know it has
  730. been stopped by an admin and not by a total update request, for instance?
  731. In any case, how is this protocol shutdown situation different from all the
  732. other ones that are present in this state machine? */
  733. /* richm: We at least need to let monitors know that the protocol has been
  734. shutdown - maybe they can figure out why */
  735. agmt_set_last_update_status(prp->agmt, 0, 0, "Protocol stopped");
  736. agmt_update_done(prp->agmt, 0);
  737. break;
  738. }
  739. agmt_set_last_update_status(prp->agmt, 0, 0, "Incremental update started");
  740. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> windows_examine_update_vector");
  741. rc = windows_examine_update_vector(prp, ruv);
  742. /*
  743. * Decide what to do next - proceed with incremental,
  744. * backoff, or total update
  745. */
  746. switch (rc)
  747. {
  748. case EXAMINE_RUV_PARAM_ERROR:
  749. /* this is really bad - we have NULL prp! */
  750. next_state = STATE_STOP_FATAL_ERROR;
  751. break;
  752. case EXAMINE_RUV_PRISTINE_REPLICA:
  753. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  754. "%s: Replica has no update vector. It has never been initialized.\n",
  755. agmt_get_long_name(prp->agmt));
  756. next_state = STATE_BACKOFF_START;
  757. break;
  758. case EXAMINE_RUV_GENERATION_MISMATCH:
  759. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  760. "%s: The remote replica has a different database generation ID than "
  761. "the local database. You may have to reinitialize the remote replica, "
  762. "or the local replica.\n", agmt_get_long_name(prp->agmt));
  763. next_state = STATE_BACKOFF_START;
  764. break;
  765. case EXAMINE_RUV_REPLICA_TOO_OLD:
  766. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  767. "%s: Replica update vector is too out of date to bring "
  768. "into sync using the incremental protocol. The replica "
  769. "must be reinitialized.\n", agmt_get_long_name(prp->agmt));
  770. next_state = STATE_BACKOFF_START;
  771. break;
  772. case EXAMINE_RUV_OK:
  773. /* update our csn generator state with the consumer's ruv data */
  774. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> windows_examine_update_vector OK");
  775. object_acquire(prp->replica_object);
  776. replica = object_get_data(prp->replica_object);
  777. rc = replica_update_csngen_state (replica, ruv);
  778. object_release (prp->replica_object);
  779. replica = NULL;
  780. if (rc == CSN_LIMIT_EXCEEDED) /* too much skew */
  781. {
  782. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  783. "%s: Incremental protocol: fatal error - too much time skew between replicas!\n",
  784. agmt_get_long_name(prp->agmt));
  785. next_state = STATE_STOP_FATAL_ERROR;
  786. }
  787. else if (rc != 0) /* internal error */
  788. {
  789. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  790. "%s: Incremental protocol: fatal internal error updating the CSN generator!\n",
  791. agmt_get_long_name(prp->agmt));
  792. next_state = STATE_STOP_FATAL_ERROR;
  793. }
  794. else
  795. {
  796. /* call begin incremental update callback */
  797. winsync_plugin_call_begin_update_cb(prp->agmt,
  798. windows_private_get_directory_subtree(prp->agmt),
  799. windows_private_get_windows_subtree(prp->agmt),
  800. 0 /* is_total == FALSE */);
  801. if ((one_way == ONE_WAY_SYNC_DISABLED) || (one_way == ONE_WAY_SYNC_TO_AD)) {
  802. rc = send_updates(prp, ruv, &num_changes_sent, 1 /* actually send updates */);
  803. } else {
  804. /* We call send_updates to fast-forward the RUV
  805. * without actually sending any updates. */
  806. rc = send_updates(prp, ruv, &num_changes_sent, 0 /* don't send updates */);
  807. }
  808. if (rc == UPDATE_NO_MORE_UPDATES)
  809. {
  810. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_NO_MORE_UPDATES -> STATE_WAIT_CHANGES");
  811. agmt_set_last_update_status(prp->agmt, 0, 0, "Incremental update succeeded");
  812. next_state = STATE_WAIT_CHANGES;
  813. }
  814. else if (rc == UPDATE_YIELD)
  815. {
  816. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_YIELD -> STATE_BACKOFF_START");
  817. agmt_set_last_update_status(prp->agmt, 0, 0, "Incremental update succeeded and yielded");
  818. next_state = STATE_BACKOFF_START;
  819. }
  820. else if (rc == UPDATE_TRANSIENT_ERROR)
  821. {
  822. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_TRANSIENT_ERROR -> STATE_BACKOFF_START");
  823. next_state = STATE_BACKOFF_START;
  824. }
  825. else if (rc == UPDATE_FATAL_ERROR)
  826. {
  827. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_FATAL_ERROR -> STATE_STOP_FATAL_ERROR");
  828. next_state = STATE_STOP_FATAL_ERROR;
  829. }
  830. else if (rc == UPDATE_SCHEDULE_WINDOW_CLOSED)
  831. {
  832. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_SCHEDULE_WINDOW_CLOSED -> STATE_WAIT_WINDOW_OPEN");
  833. /* ONREPL - I don't think we should check this. We might be
  834. here because of replicate_now event - so we don't care
  835. about the schedule */
  836. next_state = STATE_WAIT_WINDOW_OPEN;
  837. /* ONREPL - do we need to release the replica here ? */
  838. windows_conn_disconnect (prp->conn);
  839. }
  840. else if (rc == UPDATE_CONNECTION_LOST)
  841. {
  842. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_CONNECTION_LOST -> STATE_BACKOFF_START");
  843. next_state = STATE_BACKOFF_START;
  844. }
  845. else if (rc == UPDATE_TIMEOUT)
  846. {
  847. dev_debug("windows_inc_run(STATE_SENDING_UPDATES) -> send_updates = UPDATE_TIMEOUT -> STATE_BACKOFF_START");
  848. next_state = STATE_BACKOFF_START;
  849. }
  850. }
  851. last_start_time = 0UL;
  852. break;
  853. }
  854. if ( run_dirsync && ((one_way == ONE_WAY_SYNC_DISABLED) || (one_way == ONE_WAY_SYNC_FROM_AD)))
  855. {
  856. /* Don't run dirsync if we encountered
  857. * an error when sending updates to AD. */
  858. if (rc == UPDATE_NO_MORE_UPDATES) {
  859. windows_dirsync_inc_run(prp);
  860. windows_private_save_dirsync_cookie(prp->agmt);
  861. }
  862. run_dirsync = PR_FALSE;
  863. }
  864. agmt_set_last_update_end(prp->agmt, current_time());
  865. agmt_set_update_in_progress(prp->agmt, PR_FALSE);
  866. agmt_update_done(prp->agmt, 0);
  867. /* If timed out, close the connection after released the replica */
  868. windows_release_replica(prp);
  869. if (rc == UPDATE_TIMEOUT) {
  870. windows_conn_disconnect(prp->conn);
  871. }
  872. /* call end incremental update callback */
  873. winsync_plugin_call_end_update_cb(prp->agmt,
  874. windows_private_get_directory_subtree(prp->agmt),
  875. windows_private_get_windows_subtree(prp->agmt),
  876. 0 /* is_total == FALSE */);
  877. if (rc == UPDATE_NO_MORE_UPDATES && num_changes_sent > 0)
  878. {
  879. if (pausetime > 0)
  880. {
  881. /* richm - 20020219 - If we have acquired the consumer, and another master has gone
  882. into backoff waiting for us to release it, we may acquire the replica sooner
  883. than the other master has a chance to, and the other master may not be able
  884. to acquire the consumer for a long time (hours, days?) if this server is
  885. under a heavy load (see reliab06 et. al. system tests)
  886. So, this sleep gives the other master(s) a chance to acquire the consumer
  887. replica */
  888. long loops = pausetime;
  889. /* the while loop is so that we don't just sleep and sleep if an
  890. event comes in that we should handle immediately (like shutdown) */
  891. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  892. "%s: Pausing updates for %ld seconds to allow other suppliers to update consumer\n",
  893. agmt_get_long_name(prp->agmt), pausetime);
  894. while (loops-- && !(PROTOCOL_IS_SHUTDOWN(prp)))
  895. {
  896. DS_Sleep(PR_SecondsToInterval(1));
  897. }
  898. }
  899. else if (num_changes_sent > 10)
  900. {
  901. /* wait for consumer to write its ruv if the replication was busy */
  902. /* When asked, consumer sends its ruv in cache to the supplier. */
  903. /* DS_Sleep ( PR_SecondsToInterval(1) ); */
  904. }
  905. }
  906. break;
  907. case STATE_STOP_FATAL_ERROR:
  908. /*
  909. * We encountered some sort of a fatal error. Suspend.
  910. */
  911. /* XXXggood update state in replica */
  912. agmt_set_last_update_status(prp->agmt, -1, 0, "Incremental update has failed and requires administrator action");
  913. dev_debug("windows_inc_run(STATE_STOP_FATAL_ERROR)");
  914. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  915. "%s: Incremental update failed and requires administrator action\n",
  916. agmt_get_long_name(prp->agmt));
  917. next_state = STATE_STOP_FATAL_ERROR_PART2;
  918. break;
  919. case STATE_STOP_FATAL_ERROR_PART2:
  920. if (PROTOCOL_IS_SHUTDOWN(prp))
  921. {
  922. done = 1;
  923. break;
  924. }
  925. /* MAB: This state is the FATAL state where we are supposed to get
  926. as a result of a FATAL error on send_updates. But, as bug
  927. states, send_updates was always returning TRANSIENT errors and never
  928. FATAL... In other words, this code has never been tested before...
  929. As of 01/16/01, this piece of code was in a very dangerous state. In particular,
  930. 1) it does not catch any events
  931. 2) it is a terminal state (once reached it never transitions to a different state)
  932. Both things combined make this state to become a consuming infinite loop
  933. that is useless after all (we are in a fatal place requiring manual admin jobs */
  934. /* MAB: The following lines fix problem number 1 above... When the code gets
  935. into this state, it should only get a chance to get out of it by an
  936. EVENT_AGMT_CHANGED event... All other events should be ignored */
  937. else if (event_occurred(prp, EVENT_AGMT_CHANGED))
  938. {
  939. dev_debug("windows_inc_run(STATE_STOP_FATAL_ERROR): EVENT_AGMT_CHANGED received\n");
  940. /* Chance to recover for the EVENT_AGMT_CHANGED event.
  941. This is not mandatory, but fixes problem 2 above */
  942. next_state = STATE_STOP_NORMAL_TERMINATION;
  943. }
  944. else
  945. {
  946. dev_debug("windows_inc_run(STATE_STOP_FATAL_ERROR): Event received. Clearing it\n");
  947. reset_events (prp);
  948. }
  949. protocol_sleep (prp, PR_INTERVAL_NO_TIMEOUT);
  950. break;
  951. case STATE_STOP_NORMAL_TERMINATION:
  952. /*
  953. * We encountered some sort of a fatal error. Return.
  954. */
  955. /* XXXggood update state in replica */
  956. dev_debug("windows_inc_run(STATE_STOP_NORMAL_TERMINATION)");
  957. done = 1;
  958. break;
  959. }
  960. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  961. "%s: State: %s -> %s\n",
  962. agmt_get_long_name(prp->agmt),
  963. state2name(current_state), state2name(next_state));
  964. current_state = next_state;
  965. } while (!done);
  966. /* remove_protocol_callbacks(prp); */
  967. prp->stopped = 1;
  968. /* Cancel any linger timer that might be in effect... */
  969. windows_conn_cancel_linger(prp->conn);
  970. /* ... and disconnect, if currently connected */
  971. windows_conn_disconnect(prp->conn);
  972. ruv_destroy ( &ruv );
  973. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_run\n" );
  974. }
  975. /*
  976. * Go to sleep until awakened.
  977. */
  978. static void
  979. protocol_sleep(Private_Repl_Protocol *prp, PRIntervalTime duration)
  980. {
  981. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> protocol_sleep\n" );
  982. PR_ASSERT(NULL != prp);
  983. PR_Lock(prp->lock);
  984. /* we should not go to sleep if there are events available to be processed.
  985. Otherwise, we can miss the event that suppose to wake us up */
  986. if (prp->eventbits == 0)
  987. PR_WaitCondVar(prp->cvar, duration);
  988. else
  989. {
  990. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  991. "%s: Incremental protocol: can't go to sleep: event bits - %x\n",
  992. agmt_get_long_name(prp->agmt), prp->eventbits);
  993. }
  994. PR_Unlock(prp->lock);
  995. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= protocol_sleep\n" );
  996. }
  997. /*
  998. * Notify the protocol about some event. Signal the condition
  999. * variable in case the protocol is sleeping. Multiple occurences
  1000. * of a single event type are not remembered (e.g. no stack
  1001. * of events is maintained).
  1002. */
  1003. static void
  1004. event_notify(Private_Repl_Protocol *prp, PRUint32 event)
  1005. {
  1006. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> event_notify\n" );
  1007. PR_ASSERT(NULL != prp);
  1008. PR_Lock(prp->lock);
  1009. prp->eventbits |= event;
  1010. PR_NotifyCondVar(prp->cvar);
  1011. PR_Unlock(prp->lock);
  1012. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= event_notify\n" );
  1013. }
  1014. /*
  1015. * Test to see if an event occurred. The event is cleared when
  1016. * read.
  1017. */
  1018. static PRUint32
  1019. event_occurred(Private_Repl_Protocol *prp, PRUint32 event)
  1020. {
  1021. PRUint32 return_value;
  1022. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> event_occurred\n" );
  1023. PR_ASSERT(NULL != prp);
  1024. PR_Lock(prp->lock);
  1025. return_value = (prp->eventbits & event);
  1026. prp->eventbits &= ~event; /* Clear event */
  1027. PR_Unlock(prp->lock);
  1028. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= event_occurred\n" );
  1029. return return_value;
  1030. }
  1031. static void
  1032. reset_events (Private_Repl_Protocol *prp)
  1033. {
  1034. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> reset_events\n" );
  1035. PR_ASSERT(NULL != prp);
  1036. PR_Lock(prp->lock);
  1037. prp->eventbits = 0;
  1038. PR_Unlock(prp->lock);
  1039. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= reset_events\n" );
  1040. }
  1041. static PRBool
  1042. is_dummy_operation (const slapi_operation_parameters *op)
  1043. {
  1044. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> is_dummy_operation\n" );
  1045. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= is_dummy_operation\n" );
  1046. return (strcmp (op->target_address.uniqueid, START_ITERATION_ENTRY_UNIQUEID) == 0);
  1047. }
  1048. void
  1049. w_cl5_operation_parameters_done (struct slapi_operation_parameters *sop)
  1050. {
  1051. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> w_cl5_operation_parameters_done\n" );
  1052. if(sop!=NULL) {
  1053. switch(sop->operation_type)
  1054. {
  1055. case SLAPI_OPERATION_BIND:
  1056. slapi_ch_free((void **)&(sop->p.p_bind.bind_saslmechanism));
  1057. if (sop->p.p_bind.bind_creds)
  1058. ber_bvecfree((struct berval**)&(sop->p.p_bind.bind_creds));
  1059. if (sop->p.p_bind.bind_ret_saslcreds)
  1060. ber_bvecfree((struct berval**)&(sop->p.p_bind.bind_ret_saslcreds));
  1061. sop->p.p_bind.bind_creds = NULL;
  1062. sop->p.p_bind.bind_ret_saslcreds = NULL;
  1063. break;
  1064. case SLAPI_OPERATION_COMPARE:
  1065. ava_done((struct ava *)&(sop->p.p_compare.compare_ava));
  1066. break;
  1067. case SLAPI_OPERATION_SEARCH:
  1068. slapi_ch_free((void **)&(sop->p.p_search.search_strfilter));
  1069. charray_free(sop->p.p_search.search_attrs);
  1070. slapi_filter_free(sop->p.p_search.search_filter,1);
  1071. break;
  1072. case SLAPI_OPERATION_MODRDN:
  1073. sop->p.p_modrdn.modrdn_deloldrdn = 0;
  1074. break;
  1075. case SLAPI_OPERATION_EXTENDED:
  1076. slapi_ch_free((void **)&(sop->p.p_extended.exop_oid));
  1077. if (sop->p.p_extended.exop_value)
  1078. ber_bvecfree((struct berval**)&(sop->p.p_extended.exop_value));
  1079. sop->p.p_extended.exop_value = NULL;
  1080. break;
  1081. default:
  1082. break;
  1083. }
  1084. }
  1085. operation_parameters_done(sop);
  1086. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= w_cl5_operation_parameters_done\n" );
  1087. }
  1088. /*
  1089. * Send a set of updates to the replica. Assumes that (1) the replica
  1090. * has already been acquired, (2) that the consumer's update vector has
  1091. * been checked and (3) that it's ok to send incremental updates.
  1092. * Returns:
  1093. * UPDATE_NO_MORE_UPDATES - all updates were sent succussfully
  1094. * UPDATE_TRANSIENT_ERROR - some non-permanent error occurred. Try again later.
  1095. * UPDATE_FATAL_ERROR - some bad, permanent error occurred.
  1096. * UPDATE_SCHEDULE_WINDOW_CLOSED - the schedule window closed on us.
  1097. */
  1098. static int
  1099. send_updates(Private_Repl_Protocol *prp, RUV *remote_update_vector, PRUint32 *num_changes_sent, int do_send)
  1100. {
  1101. CL5Entry entry;
  1102. slapi_operation_parameters op;
  1103. int return_value = UPDATE_NO_MORE_UPDATES;
  1104. int rc;
  1105. int set_mincsn = 0;
  1106. CL5ReplayIterator *changelog_iterator = NULL;
  1107. RUV *current_ruv = ruv_dup(remote_update_vector);
  1108. CSN *mincsn = NULL;
  1109. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> send_updates\n" );
  1110. *num_changes_sent = 0;
  1111. /* Check if the min csn is set in our RUV to see if we need to set it below. */
  1112. ruv_get_min_csn(current_ruv, &mincsn);
  1113. if (!mincsn) {
  1114. set_mincsn = 1;
  1115. } else {
  1116. csn_free(&mincsn);
  1117. }
  1118. /*
  1119. * Iterate over the changelog. Retrieve each update,
  1120. * construct an appropriate LDAP operation,
  1121. * attaching the CSN, and send the change.
  1122. */
  1123. rc = cl5CreateReplayIteratorEx( prp, remote_update_vector, &changelog_iterator, agmt_get_consumerRID(prp->agmt));
  1124. if (CL5_SUCCESS != rc)
  1125. {
  1126. switch (rc)
  1127. {
  1128. case CL5_BAD_DATA: /* invalid parameter passed to the function */
  1129. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1130. "%s: Invalid parameter passed to cl5CreateReplayIterator\n",
  1131. agmt_get_long_name(prp->agmt));
  1132. return_value = UPDATE_FATAL_ERROR;
  1133. break;
  1134. case CL5_BAD_FORMAT: /* db data has unexpected format */
  1135. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1136. "%s: Unexpected format encountered in changelog database\n",
  1137. agmt_get_long_name(prp->agmt));
  1138. return_value = UPDATE_FATAL_ERROR;
  1139. break;
  1140. case CL5_BAD_STATE: /* changelog is in an incorrect state for attempted operation */
  1141. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1142. "%s: Changelog database was in an incorrect state\n",
  1143. agmt_get_long_name(prp->agmt));
  1144. return_value = UPDATE_FATAL_ERROR;
  1145. break;
  1146. case CL5_BAD_DBVERSION: /* changelog has invalid dbversion */
  1147. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1148. "%s: Incorrect dbversion found in changelog database\n",
  1149. agmt_get_long_name(prp->agmt));
  1150. return_value = UPDATE_FATAL_ERROR;
  1151. break;
  1152. case CL5_DB_ERROR: /* database error */
  1153. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1154. "%s: A changelog database error was encountered\n",
  1155. agmt_get_long_name(prp->agmt));
  1156. return_value = UPDATE_FATAL_ERROR;
  1157. break;
  1158. case CL5_NOTFOUND: /* we have no changes to send */
  1159. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1160. "%s: No changes to send\n",
  1161. agmt_get_long_name(prp->agmt));
  1162. return_value = UPDATE_NO_MORE_UPDATES;
  1163. break;
  1164. case CL5_MEMORY_ERROR: /* memory allocation failed */
  1165. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1166. "%s: Memory allocation error occurred\n",
  1167. agmt_get_long_name(prp->agmt));
  1168. return_value = UPDATE_FATAL_ERROR;
  1169. break;
  1170. case CL5_SYSTEM_ERROR: /* NSPR error occurred: use PR_GetError for furhter info */
  1171. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1172. "%s: An NSPR error (%d) occurred\n",
  1173. agmt_get_long_name(prp->agmt), PR_GetError());
  1174. return_value = UPDATE_TRANSIENT_ERROR;
  1175. break;
  1176. case CL5_CSN_ERROR: /* CSN API failed */
  1177. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1178. "%s: A CSN API failure was encountered\n",
  1179. agmt_get_long_name(prp->agmt));
  1180. return_value = UPDATE_TRANSIENT_ERROR;
  1181. break;
  1182. case CL5_RUV_ERROR: /* RUV API failed */
  1183. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1184. "%s: An RUV API failure occurred\n",
  1185. agmt_get_long_name(prp->agmt));
  1186. return_value = UPDATE_TRANSIENT_ERROR;
  1187. break;
  1188. case CL5_OBJSET_ERROR: /* namedobjset api failed */
  1189. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1190. "%s: A namedobject API failure occurred\n",
  1191. agmt_get_long_name(prp->agmt));
  1192. return_value = UPDATE_TRANSIENT_ERROR;
  1193. break;
  1194. case CL5_PURGED_DATA: /* requested data has been purged */
  1195. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1196. "%s: Data required to update replica has been purged. "
  1197. "The replica must be reinitialized.\n",
  1198. agmt_get_long_name(prp->agmt));
  1199. return_value = UPDATE_FATAL_ERROR;
  1200. break;
  1201. case CL5_MISSING_DATA: /* data should be in the changelog, but is missing */
  1202. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1203. "%s: Missing data encountered\n",
  1204. agmt_get_long_name(prp->agmt));
  1205. return_value = UPDATE_FATAL_ERROR;
  1206. break;
  1207. case CL5_UNKNOWN_ERROR: /* unclassified error */
  1208. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1209. "%s: An unknown error was ecountered\n",
  1210. agmt_get_long_name(prp->agmt));
  1211. return_value = UPDATE_TRANSIENT_ERROR;
  1212. break;
  1213. default:
  1214. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1215. "%s: An unknown error (%d) occurred "
  1216. "(cl5CreateReplayIterator)\n",
  1217. agmt_get_long_name(prp->agmt), rc);
  1218. return_value = UPDATE_TRANSIENT_ERROR;
  1219. }
  1220. }
  1221. else
  1222. {
  1223. int finished = 0;
  1224. ConnResult replay_crc;
  1225. char csn_str[CSN_STRSIZE];
  1226. memset ( (void*)&op, 0, sizeof (op) );
  1227. entry.op = &op;
  1228. do {
  1229. int mark_record_done = 0;
  1230. w_cl5_operation_parameters_done ( entry.op );
  1231. memset ( (void*)entry.op, 0, sizeof (op) );
  1232. rc = cl5GetNextOperationToReplay(changelog_iterator, &entry);
  1233. switch (rc)
  1234. {
  1235. case CL5_SUCCESS:
  1236. /* check that we don't return dummy entries */
  1237. if (is_dummy_operation (entry.op))
  1238. {
  1239. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1240. "%s: changelog iteration code returned a dummy entry with csn %s, "
  1241. "skipping ...\n",
  1242. agmt_get_long_name(prp->agmt), csn_as_string(entry.op->csn, PR_FALSE, csn_str));
  1243. continue;
  1244. }
  1245. /* This is where the work actually happens: */
  1246. if (do_send) {
  1247. replay_crc = windows_replay_update(prp, entry.op);
  1248. } else {
  1249. /* Skip sending the updates and just set a success code. */
  1250. replay_crc = CONN_OPERATION_SUCCESS;
  1251. }
  1252. if (CONN_OPERATION_SUCCESS != replay_crc)
  1253. {
  1254. int operation, error;
  1255. windows_conn_get_error(prp->conn, &operation, &error);
  1256. csn_as_string(entry.op->csn, PR_FALSE, csn_str);
  1257. /* Figure out what to do next */
  1258. if (CONN_OPERATION_FAILED == replay_crc)
  1259. {
  1260. /* Map ldap error code to return value */
  1261. if (!windows_ignore_error_and_keep_going(error))
  1262. {
  1263. return_value = UPDATE_TRANSIENT_ERROR;
  1264. finished = 1;
  1265. }
  1266. else
  1267. {
  1268. agmt_inc_last_update_changecount (prp->agmt, csn_get_replicaid(entry.op->csn), 1 /*skipped*/);
  1269. mark_record_done = 1;
  1270. }
  1271. slapi_log_error(finished ? SLAPI_LOG_FATAL : slapi_log_urp, windows_repl_plugin_name,
  1272. "%s: Consumer failed to replay change (uniqueid %s, CSN %s): %s. %s.\n",
  1273. agmt_get_long_name(prp->agmt),
  1274. entry.op->target_address.uniqueid, csn_str,
  1275. ldap_err2string(error),
  1276. finished ? "Will retry later" : "Skipping");
  1277. }
  1278. else if (CONN_NOT_CONNECTED == replay_crc)
  1279. {
  1280. /* We lost the connection - enter backoff state */
  1281. return_value = UPDATE_TRANSIENT_ERROR;
  1282. finished = 1;
  1283. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1284. "%s: Consumer failed to replay change (uniqueid %s, CSN %s): "
  1285. "%s. Will retry later.\n",
  1286. agmt_get_long_name(prp->agmt),
  1287. entry.op->target_address.uniqueid, csn_str,
  1288. error ? ldap_err2string(error) : "Connection lost");
  1289. }
  1290. else if (CONN_TIMEOUT == replay_crc)
  1291. {
  1292. return_value = UPDATE_TIMEOUT;
  1293. finished = 1;
  1294. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1295. "%s: Consumer timed out to replay change (uniqueid %s, CSN %s): "
  1296. "%s.\n",
  1297. agmt_get_long_name(prp->agmt),
  1298. entry.op->target_address.uniqueid, csn_str,
  1299. error ? ldap_err2string(error) : "Timeout");
  1300. }
  1301. else if (CONN_LOCAL_ERROR == replay_crc)
  1302. {
  1303. /*
  1304. * Something bad happened on the local server - enter
  1305. * backoff state.
  1306. */
  1307. return_value = UPDATE_TRANSIENT_ERROR;
  1308. finished = 1;
  1309. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1310. "%s: Failed to replay change (uniqueid %s, CSN %s): "
  1311. "Local error. Will retry later.\n",
  1312. agmt_get_long_name(prp->agmt),
  1313. entry.op->target_address.uniqueid, csn_str);
  1314. }
  1315. }
  1316. else
  1317. {
  1318. /* Positive response received */
  1319. (*num_changes_sent)++;
  1320. agmt_inc_last_update_changecount (prp->agmt, csn_get_replicaid(entry.op->csn), 0 /*replayed*/);
  1321. mark_record_done = 1;
  1322. }
  1323. if (mark_record_done)
  1324. {
  1325. /* If this is the very first change being sent,
  1326. * it's possible that we haven't set a min csn
  1327. * in the RUV yet. This is possible because we
  1328. * simply copy the supplier RUV during the total
  1329. * update process. The supplier RUV will not have
  1330. * a min or max csn set if no changes have ever
  1331. * been written to it's changelog. We need to set
  1332. * the min csn for the consumer here to prevent
  1333. * problems with further sync operations. */
  1334. if (set_mincsn) {
  1335. ruv_set_min_csn(current_ruv, entry.op->csn, NULL);
  1336. set_mincsn = 0;
  1337. }
  1338. /* Bring the consumers (AD) RUV up to date.
  1339. * This sets the max csn. */
  1340. ruv_force_csn_update(current_ruv, entry.op->csn);
  1341. }
  1342. break;
  1343. case CL5_BAD_DATA:
  1344. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1345. "%s: Invalid parameter passed to cl5GetNextOperationToReplay\n",
  1346. agmt_get_long_name(prp->agmt));
  1347. return_value = UPDATE_FATAL_ERROR;
  1348. finished = 1;
  1349. break;
  1350. case CL5_NOTFOUND:
  1351. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1352. "%s: No more updates to send (cl5GetNextOperationToReplay)\n",
  1353. agmt_get_long_name(prp->agmt));
  1354. return_value = UPDATE_NO_MORE_UPDATES;
  1355. finished = 1;
  1356. break;
  1357. case CL5_DB_ERROR:
  1358. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1359. "%s: A database error occurred (cl5GetNextOperationToReplay)\n",
  1360. agmt_get_long_name(prp->agmt));
  1361. return_value = UPDATE_FATAL_ERROR;
  1362. finished = 1;
  1363. break;
  1364. case CL5_BAD_FORMAT:
  1365. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1366. "%s: A malformed changelog entry was encountered (cl5GetNextOperationToReplay)\n",
  1367. agmt_get_long_name(prp->agmt));
  1368. break;
  1369. case CL5_MEMORY_ERROR:
  1370. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1371. "%s: A memory allocation error occurred (cl5GetNextOperationToRepla)\n",
  1372. agmt_get_long_name(prp->agmt));
  1373. return_value = UPDATE_FATAL_ERROR;
  1374. break;
  1375. default:
  1376. slapi_log_error(SLAPI_LOG_FATAL, windows_repl_plugin_name,
  1377. "%s: Unknown error code (%d) returned from cl5GetNextOperationToReplay\n",
  1378. agmt_get_long_name(prp->agmt), rc);
  1379. return_value = UPDATE_TRANSIENT_ERROR;
  1380. break;
  1381. }
  1382. /* Check for protocol shutdown */
  1383. if (prp->terminate)
  1384. {
  1385. return_value = UPDATE_NO_MORE_UPDATES;
  1386. finished = 1;
  1387. }
  1388. if (*num_changes_sent >= MAX_CHANGES_PER_SESSION)
  1389. {
  1390. return_value = UPDATE_YIELD;
  1391. finished = 1;
  1392. }
  1393. } while (!finished);
  1394. w_cl5_operation_parameters_done ( entry.op );
  1395. cl5DestroyReplayIterator(&changelog_iterator);
  1396. }
  1397. /* Save the RUV that we successfully replayed, this ensures that next time we start off at the next changelog record */
  1398. if (current_ruv)
  1399. {
  1400. agmt_set_consumer_ruv(prp->agmt,current_ruv);
  1401. ruv_destroy(&current_ruv);
  1402. }
  1403. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= send_updates\n" );
  1404. return return_value;
  1405. }
  1406. /*
  1407. * XXXggood this should probably be in the superclass, since the full update
  1408. * protocol is going to need it too.
  1409. */
  1410. static int
  1411. windows_inc_stop(Private_Repl_Protocol *prp)
  1412. {
  1413. int return_value;
  1414. PRIntervalTime start, maxwait, now;
  1415. int seconds = 1200;
  1416. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_stop\n" );
  1417. maxwait = PR_SecondsToInterval(seconds);
  1418. prp->terminate = 1;
  1419. event_notify(prp, EVENT_PROTOCOL_SHUTDOWN);
  1420. start = PR_IntervalNow();
  1421. now = start;
  1422. while (!prp->stopped && ((now - start) < maxwait))
  1423. {
  1424. DS_Sleep(PR_SecondsToInterval(1));
  1425. now = PR_IntervalNow();
  1426. }
  1427. if (!prp->stopped)
  1428. {
  1429. /* Isn't listening. Do something drastic. */
  1430. return_value = -1;
  1431. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1432. "%s: windows_inc_stop: protocol does not stop after %d seconds\n",
  1433. agmt_get_long_name(prp->agmt), seconds);
  1434. }
  1435. else
  1436. {
  1437. return_value = 0;
  1438. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1439. "%s: windows_inc_stop: protocol stopped after %d seconds\n",
  1440. agmt_get_long_name(prp->agmt),
  1441. PR_IntervalToSeconds(now-start));
  1442. }
  1443. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_stop\n" );
  1444. return return_value;
  1445. }
  1446. static int
  1447. windows_inc_status(Private_Repl_Protocol *prp)
  1448. {
  1449. int return_value = 0;
  1450. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_status\n" );
  1451. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_status\n" );
  1452. return return_value;
  1453. }
  1454. static void
  1455. windows_inc_notify_update(Private_Repl_Protocol *prp)
  1456. {
  1457. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_notify_update\n" );
  1458. event_notify(prp, EVENT_TRIGGERING_CRITERIA_MET);
  1459. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_notify_update\n" );
  1460. }
  1461. static void
  1462. windows_inc_update_now(Private_Repl_Protocol *prp)
  1463. {
  1464. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_update_now\n" );
  1465. event_notify(prp, EVENT_REPLICATE_NOW);
  1466. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_update_now\n" );
  1467. }
  1468. static void
  1469. windows_inc_notify_agmt_changed(Private_Repl_Protocol *prp)
  1470. {
  1471. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_notify_agmt_changed\n" );
  1472. event_notify(prp, EVENT_AGMT_CHANGED);
  1473. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_notify_agmt_changed\n" );
  1474. }
  1475. static void
  1476. windows_inc_notify_window_opened (Private_Repl_Protocol *prp)
  1477. {
  1478. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_notify_window_opened\n" );
  1479. event_notify(prp, EVENT_WINDOW_OPENED);
  1480. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_notify_window_opened\n" );
  1481. }
  1482. static void
  1483. windows_inc_notify_window_closed (Private_Repl_Protocol *prp)
  1484. {
  1485. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_notify_window_closed\n" );
  1486. event_notify(prp, EVENT_WINDOW_CLOSED);
  1487. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_notify_window_closed\n" );
  1488. }
  1489. Private_Repl_Protocol *
  1490. Windows_Inc_Protocol_new(Repl_Protocol *rp)
  1491. {
  1492. windows_inc_private *rip = NULL;
  1493. Private_Repl_Protocol *prp = (Private_Repl_Protocol *)slapi_ch_calloc(1, sizeof(Private_Repl_Protocol));
  1494. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> Windows_Inc_Protocol_new\n" );
  1495. prp->delete = windows_inc_delete;
  1496. prp->run = windows_inc_run;
  1497. prp->stop = windows_inc_stop;
  1498. prp->status = windows_inc_status;
  1499. prp->notify_update = windows_inc_notify_update;
  1500. prp->notify_agmt_changed = windows_inc_notify_agmt_changed;
  1501. prp->notify_window_opened = windows_inc_notify_window_opened;
  1502. prp->notify_window_closed = windows_inc_notify_window_closed;
  1503. prp->update_now = windows_inc_update_now;
  1504. prp->replica_object = prot_get_replica_object(rp);
  1505. if ((prp->lock = PR_NewLock()) == NULL)
  1506. {
  1507. goto loser;
  1508. }
  1509. if ((prp->cvar = PR_NewCondVar(prp->lock)) == NULL)
  1510. {
  1511. goto loser;
  1512. }
  1513. prp->stopped = 0;
  1514. prp->terminate = 0;
  1515. prp->eventbits = 0;
  1516. prp->conn = prot_get_connection(rp);
  1517. prp->agmt = prot_get_agreement(rp);
  1518. prp->last_acquire_response_code = NSDS50_REPL_REPLICA_READY;
  1519. rip = (void *)slapi_ch_malloc(sizeof(windows_inc_private));
  1520. rip->ruv = NULL;
  1521. rip->backoff = NULL;
  1522. rip->rp = rp;
  1523. prp->private = (void *)rip;
  1524. prp->replica_acquired = PR_FALSE;
  1525. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= Windows_Inc_Protocol_new\n" );
  1526. return prp;
  1527. loser:
  1528. windows_inc_delete(&prp);
  1529. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= Windows_Inc_Protocol_new (loser)\n" );
  1530. return NULL;
  1531. }
  1532. static void
  1533. windows_inc_backoff_expired(time_t timer_fire_time, void *arg)
  1534. {
  1535. Private_Repl_Protocol *prp = (Private_Repl_Protocol *)arg;
  1536. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_inc_backoff_expired\n" );
  1537. PR_ASSERT(NULL != prp);
  1538. event_notify(prp, EVENT_BACKOFF_EXPIRED);
  1539. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_inc_backoff_expired\n" );
  1540. }
  1541. /*
  1542. * Examine the update vector and determine our course of action.
  1543. * There are 3 different possibilities, plus a catch-all error:
  1544. * 1 - no update vector (ruv is NULL). The consumer's replica is
  1545. * pristine, so it needs to be initialized. Return
  1546. * EXAMINE_RUV_PRISTINE_REPLICA.
  1547. * 2 - ruv is present, but its database generation ID doesn't
  1548. * match the local generation ID. This means that either
  1549. * the local replica must be reinitialized from the remote
  1550. * replica or vice-versa. Return
  1551. * EXAMINE_RUV_GENERATION_MISMATCH.
  1552. * 3 - ruv is present, and we have all updates needed to bring
  1553. * the replica up to date using the incremental protocol.
  1554. * return EXAMINE_RUV_OK.
  1555. * 4 - parameter error. Return EXAMINE_RUV_PARAM_ERROR
  1556. */
  1557. static int
  1558. windows_examine_update_vector(Private_Repl_Protocol *prp, RUV *remote_ruv)
  1559. {
  1560. int return_value;
  1561. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> windows_examine_update_vector\n" );
  1562. PR_ASSERT(NULL != prp);
  1563. if (NULL == prp)
  1564. {
  1565. return_value = EXAMINE_RUV_PARAM_ERROR;
  1566. }
  1567. else if (NULL == remote_ruv)
  1568. {
  1569. return_value = EXAMINE_RUV_PRISTINE_REPLICA;
  1570. }
  1571. else
  1572. {
  1573. char *local_gen = NULL;
  1574. char *remote_gen = ruv_get_replica_generation(remote_ruv);
  1575. Object *local_ruv_obj;
  1576. RUV *local_ruv;
  1577. Replica *replica;
  1578. PR_ASSERT(NULL != prp->replica_object);
  1579. replica = object_get_data(prp->replica_object);
  1580. PR_ASSERT(NULL != replica);
  1581. local_ruv_obj = replica_get_ruv (replica);
  1582. if (NULL != local_ruv_obj)
  1583. {
  1584. local_ruv = (RUV*) object_get_data (local_ruv_obj);
  1585. PR_ASSERT (local_ruv);
  1586. local_gen = ruv_get_replica_generation(local_ruv);
  1587. object_release (local_ruv_obj);
  1588. }
  1589. if (NULL == remote_gen || NULL == local_gen || strcmp(remote_gen, local_gen) != 0)
  1590. {
  1591. return_value = EXAMINE_RUV_GENERATION_MISMATCH;
  1592. }
  1593. else
  1594. {
  1595. return_value = EXAMINE_RUV_OK;
  1596. }
  1597. slapi_ch_free((void**)&remote_gen);
  1598. slapi_ch_free((void**)&local_gen);
  1599. }
  1600. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= windows_examine_update_vector\n" );
  1601. return return_value;
  1602. }
  1603. /* this function converts an aquisition code to a string - for debug output */
  1604. static const char*
  1605. acquire2name (int code)
  1606. {
  1607. switch (code)
  1608. {
  1609. case ACQUIRE_SUCCESS: return "success";
  1610. case ACQUIRE_REPLICA_BUSY: return "replica_busy";
  1611. case ACQUIRE_FATAL_ERROR: return "fatal_error";
  1612. case ACQUIRE_CONSUMER_WAS_UPTODATE: return "consumer_was_uptodate";
  1613. case ACQUIRE_TRANSIENT_ERROR: return "transient_error";
  1614. default: return "invalid_code";
  1615. }
  1616. }
  1617. /* this function converts a state to its name - for debug output */
  1618. static const char*
  1619. state2name (int state)
  1620. {
  1621. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> state2name\n" );
  1622. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= state2name\n" );
  1623. switch (state)
  1624. {
  1625. case STATE_START: return "start";
  1626. case STATE_WAIT_WINDOW_OPEN: return "wait_for_window_to_open";
  1627. case STATE_WAIT_CHANGES: return "wait_for_changes";
  1628. case STATE_READY_TO_ACQUIRE: return "ready_to_acquire_replica";
  1629. case STATE_BACKOFF_START: return "start_backoff";
  1630. case STATE_BACKOFF: return "backoff";
  1631. case STATE_SENDING_UPDATES: return "sending_updates";
  1632. case STATE_STOP_FATAL_ERROR: return "stop_fatal_error";
  1633. case STATE_STOP_FATAL_ERROR_PART2: return "stop_fatal_error";
  1634. case STATE_STOP_NORMAL_TERMINATION: return "stop_normal_termination";
  1635. default: return "invalid_state";
  1636. }
  1637. }
  1638. /* this function convert s an event to its name - for debug output */
  1639. static const char*
  1640. event2name (int event)
  1641. {
  1642. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> event2name\n" );
  1643. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= event2name\n" );
  1644. switch (event)
  1645. {
  1646. case EVENT_WINDOW_OPENED: return "update_window_opened";
  1647. case EVENT_WINDOW_CLOSED: return "update_window_closed";
  1648. case EVENT_TRIGGERING_CRITERIA_MET: return "data_modified";
  1649. case EVENT_BACKOFF_EXPIRED: return "backoff_timer_expired";
  1650. case EVENT_REPLICATE_NOW: return "replicate_now";
  1651. case EVENT_PROTOCOL_SHUTDOWN: return "protocol_shutdown";
  1652. case EVENT_AGMT_CHANGED: return "agreement_changed";
  1653. case EVENT_RUN_DIRSYNC: return "run_dirsync";
  1654. default: return "invalid_event";
  1655. }
  1656. }
  1657. static void
  1658. periodic_dirsync(time_t when, void *arg)
  1659. {
  1660. LDAPDebug0Args( LDAP_DEBUG_TRACE, "=> periodic_dirsync\n" );
  1661. slapi_log_error(SLAPI_LOG_REPL, windows_repl_plugin_name,
  1662. "Running Dirsync \n");
  1663. event_notify( (Private_Repl_Protocol*) arg, EVENT_RUN_DIRSYNC);
  1664. LDAPDebug0Args( LDAP_DEBUG_TRACE, "<= periodic_dirsync\n" );
  1665. }