notifier-circuit-breaker.test.ts 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181
  1. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
  2. import type { CircuitBreakerAlertData } from "@/lib/webhook/types";
  3. describe("sendCircuitBreakerAlert", () => {
  4. const mockRedisGet = vi.fn();
  5. const mockRedisSet = vi.fn();
  6. const mockAddNotificationJob = vi.fn(async () => {});
  7. const mockAddNotificationJobForTarget = vi.fn(async () => {});
  8. beforeEach(() => {
  9. vi.resetModules();
  10. vi.doMock("@/lib/redis/client", () => ({
  11. getRedisClient: vi.fn(() => ({
  12. get: mockRedisGet,
  13. set: mockRedisSet,
  14. })),
  15. }));
  16. vi.doMock("@/repository/notifications", () => ({
  17. getNotificationSettings: vi.fn(async () => ({
  18. enabled: true,
  19. circuitBreakerEnabled: true,
  20. useLegacyMode: true,
  21. circuitBreakerWebhook: "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxx",
  22. })),
  23. }));
  24. vi.doMock("@/lib/notification/notification-queue", () => ({
  25. addNotificationJob: mockAddNotificationJob,
  26. addNotificationJobForTarget: mockAddNotificationJobForTarget,
  27. }));
  28. vi.doMock("@/lib/logger", () => ({
  29. logger: {
  30. debug: vi.fn(),
  31. info: vi.fn(),
  32. warn: vi.fn(),
  33. error: vi.fn(),
  34. trace: vi.fn(),
  35. fatal: vi.fn(),
  36. },
  37. }));
  38. });
  39. afterEach(() => {
  40. vi.clearAllMocks();
  41. });
  42. describe("dedup key with incidentSource", () => {
  43. it("should use provider dedup key when incidentSource is provider", async () => {
  44. mockRedisGet.mockResolvedValue(null); // No cached alert
  45. const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier");
  46. const data: CircuitBreakerAlertData = {
  47. providerName: "OpenAI",
  48. providerId: 1,
  49. failureCount: 5,
  50. retryAt: "2025-01-02T12:30:00Z",
  51. incidentSource: "provider",
  52. };
  53. await sendCircuitBreakerAlert(data);
  54. // Should use dedup key with provider source
  55. expect(mockRedisSet).toHaveBeenCalledWith("circuit-breaker-alert:1:provider", "1", "EX", 300);
  56. });
  57. it("should use endpoint dedup key when incidentSource is endpoint", async () => {
  58. mockRedisGet.mockResolvedValue(null);
  59. const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier");
  60. const data: CircuitBreakerAlertData = {
  61. providerName: "OpenAI",
  62. providerId: 1,
  63. failureCount: 3,
  64. retryAt: "2025-01-02T13:00:00Z",
  65. incidentSource: "endpoint",
  66. endpointId: 42,
  67. endpointUrl: "https://api.openai.com/v1",
  68. };
  69. await sendCircuitBreakerAlert(data);
  70. // Should use dedup key with endpoint source including endpointId
  71. expect(mockRedisSet).toHaveBeenCalledWith(
  72. "circuit-breaker-alert:1:endpoint:42",
  73. "1",
  74. "EX",
  75. 300
  76. );
  77. });
  78. it("should dedup independently for same provider with different sources", async () => {
  79. // Provider alert is cached
  80. mockRedisGet.mockResolvedValueOnce("1");
  81. // Endpoint alert is NOT cached
  82. mockRedisGet.mockResolvedValueOnce(null);
  83. const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier");
  84. const providerData: CircuitBreakerAlertData = {
  85. providerName: "OpenAI",
  86. providerId: 1,
  87. failureCount: 5,
  88. retryAt: "2025-01-02T12:30:00Z",
  89. incidentSource: "provider",
  90. };
  91. const endpointData: CircuitBreakerAlertData = {
  92. providerName: "OpenAI",
  93. providerId: 1,
  94. failureCount: 3,
  95. retryAt: "2025-01-02T13:00:00Z",
  96. incidentSource: "endpoint",
  97. endpointId: 42,
  98. endpointUrl: "https://api.openai.com/v1",
  99. };
  100. await sendCircuitBreakerAlert(providerData);
  101. await sendCircuitBreakerAlert(endpointData);
  102. // Provider alert should be suppressed (cached)
  103. expect(mockRedisSet).toHaveBeenCalledTimes(1);
  104. // That one call should be for endpoint source
  105. expect(mockRedisSet).toHaveBeenCalledWith(
  106. "circuit-breaker-alert:1:endpoint:42",
  107. "1",
  108. "EX",
  109. 300
  110. );
  111. });
  112. it("should default to provider source when incidentSource is undefined", async () => {
  113. mockRedisGet.mockResolvedValue(null);
  114. const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier");
  115. const data: CircuitBreakerAlertData = {
  116. providerName: "Anthropic",
  117. providerId: 2,
  118. failureCount: 3,
  119. retryAt: "2025-01-02T13:00:00Z",
  120. // incidentSource is undefined - should default to provider
  121. };
  122. await sendCircuitBreakerAlert(data);
  123. // Should use dedup key with default provider source
  124. expect(mockRedisSet).toHaveBeenCalledWith("circuit-breaker-alert:2:provider", "1", "EX", 300);
  125. });
  126. it("should suppress endpoint alert when same endpointId was recently alerted", async () => {
  127. // First call: not cached
  128. mockRedisGet.mockResolvedValueOnce(null);
  129. // Second call: cached
  130. mockRedisGet.mockResolvedValueOnce("1");
  131. const { sendCircuitBreakerAlert } = await import("@/lib/notification/notifier");
  132. const data: CircuitBreakerAlertData = {
  133. providerName: "OpenAI",
  134. providerId: 1,
  135. failureCount: 3,
  136. retryAt: "2025-01-02T13:00:00Z",
  137. incidentSource: "endpoint",
  138. endpointId: 42,
  139. };
  140. await sendCircuitBreakerAlert(data);
  141. await sendCircuitBreakerAlert(data);
  142. // Only first call should have set cache
  143. expect(mockRedisSet).toHaveBeenCalledTimes(1);
  144. // Should have checked cache twice
  145. expect(mockRedisGet).toHaveBeenCalledTimes(2);
  146. });
  147. });
  148. });