endpoint-circuit-breaker.test.ts 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657
  1. import { afterEach, describe, expect, test, vi } from "vitest";
  2. type SavedEndpointCircuitState = {
  3. failureCount: number;
  4. lastFailureTime: number | null;
  5. circuitState: "closed" | "open" | "half-open";
  6. circuitOpenUntil: number | null;
  7. halfOpenSuccessCount: number;
  8. };
  9. function createLoggerMock() {
  10. return {
  11. debug: vi.fn(),
  12. info: vi.fn(),
  13. warn: vi.fn(),
  14. trace: vi.fn(),
  15. error: vi.fn(),
  16. fatal: vi.fn(),
  17. };
  18. }
  19. async function flushPromises(rounds = 2): Promise<void> {
  20. for (let i = 0; i < rounds; i++) {
  21. await new Promise((resolve) => setTimeout(resolve, 0));
  22. }
  23. }
  24. async function waitForMockCalled(mock: ReturnType<typeof vi.fn>, timeoutMs = 5000): Promise<void> {
  25. const startedAt = Date.now();
  26. while (mock.mock.calls.length === 0 && Date.now() - startedAt < timeoutMs) {
  27. await new Promise<void>((resolve) => setTimeout(resolve, 0));
  28. }
  29. }
  30. afterEach(() => {
  31. vi.useRealTimers();
  32. delete process.env.ENDPOINT_CIRCUIT_HEALTH_CACHE_MAX_SIZE;
  33. });
  34. describe("endpoint-circuit-breaker", () => {
  35. test("达到阈值后应打开熔断;到期后进入 half-open;成功后关闭并清零", async () => {
  36. vi.resetModules();
  37. let redisState: SavedEndpointCircuitState | null = null;
  38. const loadMock = vi.fn(async () => redisState);
  39. const saveMock = vi.fn(async (_endpointId: number, state: SavedEndpointCircuitState) => {
  40. redisState = state;
  41. });
  42. const deleteMock = vi.fn(async () => {
  43. redisState = null;
  44. });
  45. vi.doMock("@/lib/config/env.schema", () => ({
  46. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: true }),
  47. }));
  48. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  49. const sendAlertMock = vi.fn(async () => {});
  50. vi.doMock("@/lib/notification/notifier", () => ({
  51. sendCircuitBreakerAlert: sendAlertMock,
  52. }));
  53. vi.doMock("@/repository", () => ({
  54. findProviderEndpointById: vi.fn(async () => null),
  55. }));
  56. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  57. loadEndpointCircuitState: loadMock,
  58. saveEndpointCircuitState: saveMock,
  59. deleteEndpointCircuitState: deleteMock,
  60. }));
  61. vi.useFakeTimers();
  62. vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
  63. const {
  64. isEndpointCircuitOpen,
  65. getEndpointHealthInfo,
  66. recordEndpointFailure,
  67. recordEndpointSuccess,
  68. resetEndpointCircuit,
  69. } = await import("@/lib/endpoint-circuit-breaker");
  70. await recordEndpointFailure(1, new Error("boom"));
  71. await recordEndpointFailure(1, new Error("boom"));
  72. await recordEndpointFailure(1, new Error("boom"));
  73. const openState = saveMock.mock.calls[
  74. saveMock.mock.calls.length - 1
  75. ]?.[1] as SavedEndpointCircuitState;
  76. expect(openState.circuitState).toBe("open");
  77. expect(openState.failureCount).toBe(3);
  78. expect(openState.circuitOpenUntil).toBe(Date.now() + 300000);
  79. // Prime env module cache: under fake timers, dynamic import() inside isEndpointCircuitOpen
  80. // may fail to resolve the vi.doMock unless the module is already in the import cache.
  81. await import("@/lib/config/env.schema");
  82. expect(await isEndpointCircuitOpen(1)).toBe(true);
  83. vi.advanceTimersByTime(300000 + 1);
  84. expect(await isEndpointCircuitOpen(1)).toBe(false);
  85. const halfOpenState = saveMock.mock.calls[
  86. saveMock.mock.calls.length - 1
  87. ]?.[1] as SavedEndpointCircuitState;
  88. expect(halfOpenState.circuitState).toBe("half-open");
  89. await recordEndpointSuccess(1);
  90. expect(deleteMock).toHaveBeenCalledWith(1);
  91. const { health: afterSuccess } = await getEndpointHealthInfo(1);
  92. expect(afterSuccess.circuitState).toBe("closed");
  93. expect(afterSuccess.failureCount).toBe(0);
  94. expect(afterSuccess.circuitOpenUntil).toBeNull();
  95. expect(afterSuccess.lastFailureTime).toBeNull();
  96. expect(afterSuccess.halfOpenSuccessCount).toBe(0);
  97. expect(await isEndpointCircuitOpen(1)).toBe(false);
  98. const deleteCallsAfterSuccess = deleteMock.mock.calls.length;
  99. await resetEndpointCircuit(1);
  100. expect(deleteMock.mock.calls.length).toBeGreaterThan(deleteCallsAfterSuccess);
  101. // 说明:recordEndpointFailure 在达到阈值后会触发异步告警(dynamic import + await)。
  102. // 在 CI/bun 环境下,告警 Promise 可能在下一个测试开始后才完成,从而“借用”后续用例的 module mock,
  103. // 导致 sendAlertMock 被额外调用而产生偶发失败。这里用真实计时器让事件循环前进,确保告警任务尽快落地。
  104. vi.useRealTimers();
  105. await waitForMockCalled(sendAlertMock);
  106. expect(sendAlertMock).toHaveBeenCalledTimes(1);
  107. });
  108. test("recordEndpointSuccess: closed 且 failureCount>0 时应清零", async () => {
  109. vi.resetModules();
  110. const saveMock = vi.fn(async () => {});
  111. const deleteMock = vi.fn(async () => {});
  112. vi.doMock("@/lib/config/env.schema", () => ({
  113. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: true }),
  114. }));
  115. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  116. vi.useFakeTimers();
  117. vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
  118. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  119. loadEndpointCircuitState: vi.fn(async () => null),
  120. saveEndpointCircuitState: saveMock,
  121. deleteEndpointCircuitState: deleteMock,
  122. }));
  123. const { recordEndpointFailure, recordEndpointSuccess, getEndpointHealthInfo } = await import(
  124. "@/lib/endpoint-circuit-breaker"
  125. );
  126. await recordEndpointFailure(2, new Error("boom"));
  127. await recordEndpointSuccess(2);
  128. const { health } = await getEndpointHealthInfo(2);
  129. expect(health.failureCount).toBe(0);
  130. expect(health.circuitState).toBe("closed");
  131. expect(saveMock).toHaveBeenCalledTimes(1);
  132. expect(deleteMock).toHaveBeenCalledWith(2);
  133. });
  134. test("getAllEndpointHealthStatusAsync: forceRefresh 时应同步 Redis 中的计数(即使 circuitState 未变化)", async () => {
  135. vi.resetModules();
  136. const endpointId = 42;
  137. const redisStates = new Map<number, SavedEndpointCircuitState>();
  138. const loadManyMock = vi.fn(async (endpointIds: number[]) => {
  139. const result = new Map<number, SavedEndpointCircuitState>();
  140. for (const id of endpointIds) {
  141. const state = redisStates.get(id);
  142. if (state) {
  143. result.set(id, state);
  144. }
  145. }
  146. return result;
  147. });
  148. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  149. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  150. loadEndpointCircuitState: vi.fn(async () => null),
  151. loadEndpointCircuitStates: loadManyMock,
  152. saveEndpointCircuitState: vi.fn(async () => {}),
  153. deleteEndpointCircuitState: vi.fn(async () => {}),
  154. }));
  155. vi.useFakeTimers();
  156. vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
  157. const t0 = Date.now();
  158. redisStates.set(endpointId, {
  159. failureCount: 1,
  160. lastFailureTime: t0 - 1000,
  161. circuitState: "closed",
  162. circuitOpenUntil: null,
  163. halfOpenSuccessCount: 0,
  164. });
  165. const { getAllEndpointHealthStatusAsync } = await import("@/lib/endpoint-circuit-breaker");
  166. const first = await getAllEndpointHealthStatusAsync([endpointId], { forceRefresh: true });
  167. expect(first[endpointId]).toMatchObject({
  168. failureCount: 1,
  169. lastFailureTime: t0 - 1000,
  170. circuitState: "closed",
  171. circuitOpenUntil: null,
  172. halfOpenSuccessCount: 0,
  173. });
  174. redisStates.set(endpointId, {
  175. failureCount: 2,
  176. lastFailureTime: t0 + 123,
  177. circuitState: "closed",
  178. circuitOpenUntil: null,
  179. halfOpenSuccessCount: 0,
  180. });
  181. const second = await getAllEndpointHealthStatusAsync([endpointId], { forceRefresh: true });
  182. expect(second[endpointId]).toMatchObject({
  183. failureCount: 2,
  184. lastFailureTime: t0 + 123,
  185. circuitState: "closed",
  186. circuitOpenUntil: null,
  187. halfOpenSuccessCount: 0,
  188. });
  189. expect(loadManyMock).toHaveBeenCalledTimes(2);
  190. });
  191. test("getAllEndpointHealthStatusAsync: 并发请求应复用 in-flight Redis 批量加载", async () => {
  192. vi.resetModules();
  193. const loadManyMock = vi.fn(async (_endpointIds: number[]) => {
  194. await new Promise((resolve) => setTimeout(resolve, 10));
  195. const result = new Map<number, SavedEndpointCircuitState>();
  196. result.set(1, {
  197. failureCount: 0,
  198. lastFailureTime: null,
  199. circuitState: "closed",
  200. circuitOpenUntil: null,
  201. halfOpenSuccessCount: 0,
  202. });
  203. result.set(2, {
  204. failureCount: 0,
  205. lastFailureTime: null,
  206. circuitState: "closed",
  207. circuitOpenUntil: null,
  208. halfOpenSuccessCount: 0,
  209. });
  210. result.set(3, {
  211. failureCount: 0,
  212. lastFailureTime: null,
  213. circuitState: "closed",
  214. circuitOpenUntil: null,
  215. halfOpenSuccessCount: 0,
  216. });
  217. return result;
  218. });
  219. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  220. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  221. loadEndpointCircuitState: vi.fn(async () => null),
  222. loadEndpointCircuitStates: loadManyMock,
  223. saveEndpointCircuitState: vi.fn(async () => {}),
  224. deleteEndpointCircuitState: vi.fn(async () => {}),
  225. }));
  226. vi.useFakeTimers();
  227. vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
  228. const { getAllEndpointHealthStatusAsync } = await import("@/lib/endpoint-circuit-breaker");
  229. const p1 = getAllEndpointHealthStatusAsync([1, 2, 3], { forceRefresh: true });
  230. const p2 = getAllEndpointHealthStatusAsync([1, 2, 3], { forceRefresh: true });
  231. vi.advanceTimersByTime(20);
  232. await Promise.all([p1, p2]);
  233. expect(loadManyMock).toHaveBeenCalledTimes(1);
  234. });
  235. test("triggerEndpointCircuitBreakerAlert should call sendCircuitBreakerAlert", async () => {
  236. vi.resetModules();
  237. vi.doMock("@/lib/config/env.schema", () => ({
  238. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: true }),
  239. }));
  240. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  241. vi.doMock("@/lib/notification/notifier", () => ({
  242. sendCircuitBreakerAlert: vi.fn(async () => {}),
  243. }));
  244. vi.doMock("@/repository", () => ({
  245. findProviderEndpointById: vi.fn(async () => null),
  246. }));
  247. // recordEndpointFailure 会 non-blocking 触发告警;先让 event-loop 跑完再清空计数,避免串台导致误判
  248. await flushPromises();
  249. // Prime module cache for dynamic import() consumers
  250. await import("@/lib/config/env.schema");
  251. const notifierModule = await import("@/lib/notification/notifier");
  252. const sendAlertSpy = vi
  253. .spyOn(notifierModule, "sendCircuitBreakerAlert")
  254. .mockResolvedValue(undefined);
  255. sendAlertSpy.mockClear();
  256. const { triggerEndpointCircuitBreakerAlert } = await import("@/lib/endpoint-circuit-breaker");
  257. await triggerEndpointCircuitBreakerAlert(
  258. 5,
  259. 3,
  260. "2026-01-01T00:05:00.000Z",
  261. "connection refused"
  262. );
  263. const endpoint5Calls = sendAlertSpy.mock.calls
  264. .map((call) => call[0] as Record<string, unknown>)
  265. .filter((payload) => payload.endpointId === 5);
  266. expect(endpoint5Calls).toHaveLength(1);
  267. expect(endpoint5Calls[0]).toEqual({
  268. providerId: 0,
  269. providerName: "endpoint:5",
  270. failureCount: 3,
  271. retryAt: "2026-01-01T00:05:00.000Z",
  272. lastError: "connection refused",
  273. incidentSource: "endpoint",
  274. endpointId: 5,
  275. endpointUrl: undefined,
  276. });
  277. });
  278. test("triggerEndpointCircuitBreakerAlert should include endpointUrl when available", async () => {
  279. vi.resetModules();
  280. vi.doMock("@/lib/config/env.schema", () => ({
  281. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: true }),
  282. }));
  283. vi.doMock("@/lib/notification/notifier", () => ({
  284. sendCircuitBreakerAlert: vi.fn(async () => {}),
  285. }));
  286. vi.doMock("@/repository", () => ({
  287. findProviderEndpointById: vi.fn(async () => ({
  288. id: 10,
  289. url: "https://custom.example.com/v1/chat/completions",
  290. vendorId: 1,
  291. providerType: "openai",
  292. label: "Custom Endpoint",
  293. sortOrder: 0,
  294. isEnabled: true,
  295. lastProbedAt: null,
  296. lastProbeOk: null,
  297. lastProbeStatusCode: null,
  298. lastProbeLatencyMs: null,
  299. lastProbeErrorType: null,
  300. lastProbeErrorMessage: null,
  301. createdAt: new Date(),
  302. updatedAt: new Date(),
  303. deletedAt: null,
  304. })),
  305. }));
  306. // recordEndpointFailure 会 non-blocking 触发告警;先让 event-loop 跑完再清空计数,避免串台导致误判
  307. await flushPromises();
  308. // Prime module cache for dynamic import() consumers
  309. await import("@/lib/config/env.schema");
  310. const notifierModule = await import("@/lib/notification/notifier");
  311. const sendAlertSpy = vi
  312. .spyOn(notifierModule, "sendCircuitBreakerAlert")
  313. .mockResolvedValue(undefined);
  314. sendAlertSpy.mockClear();
  315. const { triggerEndpointCircuitBreakerAlert } = await import("@/lib/endpoint-circuit-breaker");
  316. await triggerEndpointCircuitBreakerAlert(10, 3, "2026-01-01T00:05:00.000Z", "timeout");
  317. const endpoint10Calls = sendAlertSpy.mock.calls
  318. .map((call) => call[0] as Record<string, unknown>)
  319. .filter((payload) => payload.endpointId === 10);
  320. expect(endpoint10Calls).toHaveLength(1);
  321. expect(endpoint10Calls[0]).toEqual({
  322. providerId: 1,
  323. providerName: "Custom Endpoint",
  324. failureCount: 3,
  325. retryAt: "2026-01-01T00:05:00.000Z",
  326. lastError: "timeout",
  327. incidentSource: "endpoint",
  328. endpointId: 10,
  329. endpointUrl: "https://custom.example.com/v1/chat/completions",
  330. });
  331. });
  332. test("recordEndpointFailure should NOT reset circuitOpenUntil when already open", async () => {
  333. vi.resetModules();
  334. let redisState: SavedEndpointCircuitState | null = null;
  335. const saveMock = vi.fn(async (_endpointId: number, state: SavedEndpointCircuitState) => {
  336. redisState = state;
  337. });
  338. vi.doMock("@/lib/config/env.schema", () => ({
  339. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: true }),
  340. }));
  341. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  342. const sendAlertMock = vi.fn(async () => {});
  343. vi.doMock("@/lib/notification/notifier", () => ({
  344. sendCircuitBreakerAlert: sendAlertMock,
  345. }));
  346. vi.doMock("@/repository", () => ({
  347. findProviderEndpointById: vi.fn(async () => null),
  348. }));
  349. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  350. loadEndpointCircuitState: vi.fn(async () => redisState),
  351. saveEndpointCircuitState: saveMock,
  352. deleteEndpointCircuitState: vi.fn(async () => {}),
  353. }));
  354. vi.useFakeTimers();
  355. vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
  356. const { recordEndpointFailure, isEndpointCircuitOpen, getEndpointHealthInfo } = await import(
  357. "@/lib/endpoint-circuit-breaker"
  358. );
  359. // Record 3 failures to open the circuit
  360. await recordEndpointFailure(100, new Error("fail"));
  361. await recordEndpointFailure(100, new Error("fail"));
  362. await recordEndpointFailure(100, new Error("fail"));
  363. // Verify circuit was opened (also serves as async flush before isEndpointCircuitOpen)
  364. const { health: healthSnap } = await getEndpointHealthInfo(100);
  365. expect(healthSnap.circuitState).toBe("open");
  366. // Prime the env module cache: under fake timers, the dynamic import("@/lib/config/env.schema")
  367. // inside isEndpointCircuitOpen may fail to resolve the mock unless the module is already cached.
  368. const envMod = await import("@/lib/config/env.schema");
  369. expect(envMod.getEnvConfig().ENABLE_ENDPOINT_CIRCUIT_BREAKER).toBe(true);
  370. expect(await isEndpointCircuitOpen(100)).toBe(true);
  371. const originalOpenUntil = redisState!.circuitOpenUntil;
  372. expect(originalOpenUntil).toBe(Date.now() + 300000);
  373. // Advance 1 min and record another failure — timer must NOT reset
  374. vi.advanceTimersByTime(60_000);
  375. await recordEndpointFailure(100, new Error("fail again"));
  376. expect(redisState!.circuitState).toBe("open");
  377. expect(redisState!.circuitOpenUntil).toBe(originalOpenUntil); // unchanged!
  378. expect(redisState!.failureCount).toBe(4);
  379. // recordEndpointFailure 在打开熔断时会 non-blocking 触发告警;避免告警任务跨测试“借用”后续 mock。
  380. vi.useRealTimers();
  381. await waitForMockCalled(sendAlertMock);
  382. expect(sendAlertMock).toHaveBeenCalledTimes(1);
  383. });
  384. test("getEndpointCircuitStateSync returns correct state for known and unknown endpoints", async () => {
  385. vi.resetModules();
  386. vi.doMock("@/lib/config/env.schema", () => ({
  387. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: true }),
  388. }));
  389. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  390. const sendAlertMock = vi.fn(async () => {});
  391. vi.doMock("@/lib/notification/notifier", () => ({
  392. sendCircuitBreakerAlert: sendAlertMock,
  393. }));
  394. vi.doMock("@/repository", () => ({
  395. findProviderEndpointById: vi.fn(async () => null),
  396. }));
  397. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  398. loadEndpointCircuitState: vi.fn(async () => null),
  399. saveEndpointCircuitState: vi.fn(async () => {}),
  400. deleteEndpointCircuitState: vi.fn(async () => {}),
  401. }));
  402. const { getEndpointCircuitStateSync, recordEndpointFailure } = await import(
  403. "@/lib/endpoint-circuit-breaker"
  404. );
  405. // Unknown endpoint returns "closed"
  406. expect(getEndpointCircuitStateSync(9999)).toBe("closed");
  407. // After opening the circuit, sync accessor reflects "open"
  408. await recordEndpointFailure(200, new Error("a"));
  409. await recordEndpointFailure(200, new Error("b"));
  410. await recordEndpointFailure(200, new Error("c"));
  411. expect(getEndpointCircuitStateSync(200)).toBe("open");
  412. // 打开熔断会触发异步告警;确保该任务在用例结束前完成,避免串台。
  413. vi.useRealTimers();
  414. await waitForMockCalled(sendAlertMock);
  415. expect(sendAlertMock).toHaveBeenCalledTimes(1);
  416. });
  417. describe("ENABLE_ENDPOINT_CIRCUIT_BREAKER disabled", () => {
  418. test("isEndpointCircuitOpen returns false when ENABLE_ENDPOINT_CIRCUIT_BREAKER=false", async () => {
  419. vi.resetModules();
  420. vi.doMock("@/lib/config/env.schema", () => ({
  421. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: false }),
  422. }));
  423. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  424. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  425. loadEndpointCircuitState: vi.fn(async () => null),
  426. saveEndpointCircuitState: vi.fn(async () => {}),
  427. deleteEndpointCircuitState: vi.fn(async () => {}),
  428. }));
  429. const { isEndpointCircuitOpen } = await import("@/lib/endpoint-circuit-breaker");
  430. expect(await isEndpointCircuitOpen(1)).toBe(false);
  431. expect(await isEndpointCircuitOpen(999)).toBe(false);
  432. });
  433. test("recordEndpointFailure is no-op when disabled", async () => {
  434. vi.resetModules();
  435. const saveMock = vi.fn(async () => {});
  436. vi.doMock("@/lib/config/env.schema", () => ({
  437. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: false }),
  438. }));
  439. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  440. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  441. loadEndpointCircuitState: vi.fn(async () => null),
  442. saveEndpointCircuitState: saveMock,
  443. deleteEndpointCircuitState: vi.fn(async () => {}),
  444. }));
  445. const { recordEndpointFailure } = await import("@/lib/endpoint-circuit-breaker");
  446. await recordEndpointFailure(1, new Error("boom"));
  447. await recordEndpointFailure(1, new Error("boom"));
  448. await recordEndpointFailure(1, new Error("boom"));
  449. expect(saveMock).not.toHaveBeenCalled();
  450. });
  451. test("recordEndpointSuccess is no-op when disabled", async () => {
  452. vi.resetModules();
  453. const saveMock = vi.fn(async () => {});
  454. vi.doMock("@/lib/config/env.schema", () => ({
  455. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: false }),
  456. }));
  457. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  458. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  459. loadEndpointCircuitState: vi.fn(async () => null),
  460. saveEndpointCircuitState: saveMock,
  461. deleteEndpointCircuitState: vi.fn(async () => {}),
  462. }));
  463. const { recordEndpointSuccess } = await import("@/lib/endpoint-circuit-breaker");
  464. await recordEndpointSuccess(1);
  465. expect(saveMock).not.toHaveBeenCalled();
  466. });
  467. test("triggerEndpointCircuitBreakerAlert is no-op when disabled", async () => {
  468. vi.resetModules();
  469. const sendAlertMock = vi.fn(async () => {});
  470. vi.doMock("@/lib/config/env.schema", () => ({
  471. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: false }),
  472. }));
  473. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  474. vi.doMock("@/lib/notification/notifier", () => ({
  475. sendCircuitBreakerAlert: sendAlertMock,
  476. }));
  477. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  478. loadEndpointCircuitState: vi.fn(async () => null),
  479. saveEndpointCircuitState: vi.fn(async () => {}),
  480. deleteEndpointCircuitState: vi.fn(async () => {}),
  481. }));
  482. const { triggerEndpointCircuitBreakerAlert } = await import("@/lib/endpoint-circuit-breaker");
  483. await triggerEndpointCircuitBreakerAlert(
  484. 5,
  485. 3,
  486. "2026-01-01T00:05:00.000Z",
  487. "connection refused"
  488. );
  489. expect(sendAlertMock).not.toHaveBeenCalled();
  490. });
  491. test("initEndpointCircuitBreaker clears in-memory state and Redis keys when disabled", async () => {
  492. vi.resetModules();
  493. const redisMock = {
  494. scan: vi
  495. .fn()
  496. .mockResolvedValueOnce([
  497. "0",
  498. ["endpoint_circuit_breaker:state:1", "endpoint_circuit_breaker:state:2"],
  499. ]),
  500. del: vi.fn(async () => {}),
  501. };
  502. vi.doMock("@/lib/config/env.schema", () => ({
  503. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: false }),
  504. }));
  505. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  506. vi.doMock("@/lib/redis/client", () => ({
  507. getRedisClient: () => redisMock,
  508. }));
  509. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  510. loadEndpointCircuitState: vi.fn(async () => null),
  511. saveEndpointCircuitState: vi.fn(async () => {}),
  512. deleteEndpointCircuitState: vi.fn(async () => {}),
  513. }));
  514. const { initEndpointCircuitBreaker } = await import("@/lib/endpoint-circuit-breaker");
  515. await initEndpointCircuitBreaker();
  516. expect(redisMock.scan).toHaveBeenCalled();
  517. expect(redisMock.del).toHaveBeenCalledWith(
  518. "endpoint_circuit_breaker:state:1",
  519. "endpoint_circuit_breaker:state:2"
  520. );
  521. });
  522. test("initEndpointCircuitBreaker is no-op when enabled", async () => {
  523. vi.resetModules();
  524. const redisMock = {
  525. scan: vi.fn(),
  526. del: vi.fn(),
  527. };
  528. vi.doMock("@/lib/config/env.schema", () => ({
  529. getEnvConfig: () => ({ ENABLE_ENDPOINT_CIRCUIT_BREAKER: true }),
  530. }));
  531. vi.doMock("@/lib/logger", () => ({ logger: createLoggerMock() }));
  532. vi.doMock("@/lib/redis/client", () => ({
  533. getRedisClient: () => redisMock,
  534. }));
  535. vi.doMock("@/lib/redis/endpoint-circuit-breaker-state", () => ({
  536. loadEndpointCircuitState: vi.fn(async () => null),
  537. saveEndpointCircuitState: vi.fn(async () => {}),
  538. deleteEndpointCircuitState: vi.fn(async () => {}),
  539. }));
  540. const { initEndpointCircuitBreaker } = await import("@/lib/endpoint-circuit-breaker");
  541. await initEndpointCircuitBreaker();
  542. expect(redisMock.scan).not.toHaveBeenCalled();
  543. expect(redisMock.del).not.toHaveBeenCalled();
  544. });
  545. });
  546. });