anthropic-provider-overrides.test.ts 36 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154
  1. import { describe, expect, it } from "vitest";
  2. import {
  3. applyAnthropicProviderOverrides,
  4. applyAnthropicProviderOverridesWithAudit,
  5. } from "@/lib/anthropic/provider-overrides";
  6. describe("Anthropic Provider Overrides", () => {
  7. describe("Provider type filtering", () => {
  8. it("should return unchanged request for non-claude/claude-auth providers (codex)", () => {
  9. const provider = {
  10. providerType: "codex",
  11. anthropicMaxTokensPreference: "32000",
  12. anthropicThinkingBudgetPreference: "10240",
  13. };
  14. const input: Record<string, unknown> = {
  15. model: "claude-3-opus-20240229",
  16. messages: [],
  17. max_tokens: 8000,
  18. };
  19. const output = applyAnthropicProviderOverrides(provider, input);
  20. expect(output).toBe(input);
  21. expect(output).toEqual(input);
  22. });
  23. it("should return unchanged request for non-claude/claude-auth providers (gemini)", () => {
  24. const provider = {
  25. providerType: "gemini",
  26. anthropicMaxTokensPreference: "32000",
  27. anthropicThinkingBudgetPreference: "10240",
  28. };
  29. const input: Record<string, unknown> = {
  30. model: "gemini-pro",
  31. messages: [],
  32. max_tokens: 8000,
  33. };
  34. const output = applyAnthropicProviderOverrides(provider, input);
  35. expect(output).toBe(input);
  36. });
  37. it("should return unchanged request for non-claude/claude-auth providers (openai-compatible)", () => {
  38. const provider = {
  39. providerType: "openai-compatible",
  40. anthropicMaxTokensPreference: "16000",
  41. };
  42. const input: Record<string, unknown> = {
  43. model: "gpt-4",
  44. messages: [],
  45. };
  46. const output = applyAnthropicProviderOverrides(provider, input);
  47. expect(output).toBe(input);
  48. });
  49. it("should apply overrides for 'claude' provider type", () => {
  50. const provider = {
  51. providerType: "claude",
  52. anthropicMaxTokensPreference: "32000",
  53. };
  54. const input: Record<string, unknown> = {
  55. model: "claude-3-opus-20240229",
  56. messages: [],
  57. max_tokens: 8000,
  58. };
  59. const output = applyAnthropicProviderOverrides(provider, input);
  60. expect(output.max_tokens).toBe(32000);
  61. });
  62. it("should apply overrides for 'claude-auth' provider type", () => {
  63. const provider = {
  64. providerType: "claude-auth",
  65. anthropicMaxTokensPreference: "16000",
  66. };
  67. const input: Record<string, unknown> = {
  68. model: "claude-3-sonnet-20240229",
  69. messages: [],
  70. max_tokens: 4000,
  71. };
  72. const output = applyAnthropicProviderOverrides(provider, input);
  73. expect(output.max_tokens).toBe(16000);
  74. });
  75. });
  76. describe("max_tokens override", () => {
  77. it("should not change request when preference is 'inherit'", () => {
  78. const provider = {
  79. providerType: "claude",
  80. anthropicMaxTokensPreference: "inherit",
  81. };
  82. const input: Record<string, unknown> = {
  83. model: "claude-3-opus-20240229",
  84. messages: [],
  85. max_tokens: 8000,
  86. };
  87. const snapshot = structuredClone(input);
  88. const output = applyAnthropicProviderOverrides(provider, input);
  89. expect(output).toEqual(snapshot);
  90. });
  91. it("should not change request when preference is null", () => {
  92. const provider = {
  93. providerType: "claude",
  94. anthropicMaxTokensPreference: null,
  95. };
  96. const input: Record<string, unknown> = {
  97. model: "claude-3-opus-20240229",
  98. messages: [],
  99. max_tokens: 8000,
  100. };
  101. const snapshot = structuredClone(input);
  102. const output = applyAnthropicProviderOverrides(provider, input);
  103. expect(output).toEqual(snapshot);
  104. });
  105. it("should not change request when preference is undefined", () => {
  106. const provider = {
  107. providerType: "claude",
  108. anthropicMaxTokensPreference: undefined,
  109. };
  110. const input: Record<string, unknown> = {
  111. model: "claude-3-opus-20240229",
  112. messages: [],
  113. max_tokens: 8000,
  114. };
  115. const snapshot = structuredClone(input);
  116. const output = applyAnthropicProviderOverrides(provider, input);
  117. expect(output).toEqual(snapshot);
  118. });
  119. it("should set max_tokens to numeric value when preference is valid string '32000'", () => {
  120. const provider = {
  121. providerType: "claude",
  122. anthropicMaxTokensPreference: "32000",
  123. };
  124. const input: Record<string, unknown> = {
  125. model: "claude-3-opus-20240229",
  126. messages: [],
  127. };
  128. const output = applyAnthropicProviderOverrides(provider, input);
  129. expect(output.max_tokens).toBe(32000);
  130. });
  131. it("should overwrite existing max_tokens value", () => {
  132. const provider = {
  133. providerType: "claude",
  134. anthropicMaxTokensPreference: "64000",
  135. };
  136. const input: Record<string, unknown> = {
  137. model: "claude-3-opus-20240229",
  138. messages: [],
  139. max_tokens: 4000,
  140. };
  141. const output = applyAnthropicProviderOverrides(provider, input);
  142. expect(output.max_tokens).toBe(64000);
  143. expect(input.max_tokens).toBe(4000);
  144. });
  145. it("should not change max_tokens for invalid numeric string", () => {
  146. const provider = {
  147. providerType: "claude",
  148. anthropicMaxTokensPreference: "invalid",
  149. };
  150. const input: Record<string, unknown> = {
  151. model: "claude-3-opus-20240229",
  152. messages: [],
  153. max_tokens: 8000,
  154. };
  155. const snapshot = structuredClone(input);
  156. const output = applyAnthropicProviderOverrides(provider, input);
  157. expect(output).toEqual(snapshot);
  158. });
  159. });
  160. describe("thinking.budget_tokens override", () => {
  161. it("should not change request when preference is 'inherit'", () => {
  162. const provider = {
  163. providerType: "claude",
  164. anthropicThinkingBudgetPreference: "inherit",
  165. };
  166. const input: Record<string, unknown> = {
  167. model: "claude-3-opus-20240229",
  168. messages: [],
  169. thinking: { type: "enabled", budget_tokens: 5000 },
  170. };
  171. const snapshot = structuredClone(input);
  172. const output = applyAnthropicProviderOverrides(provider, input);
  173. expect(output).toEqual(snapshot);
  174. });
  175. it("should not change request when preference is null", () => {
  176. const provider = {
  177. providerType: "claude",
  178. anthropicThinkingBudgetPreference: null,
  179. };
  180. const input: Record<string, unknown> = {
  181. model: "claude-3-opus-20240229",
  182. messages: [],
  183. thinking: { type: "enabled", budget_tokens: 5000 },
  184. };
  185. const snapshot = structuredClone(input);
  186. const output = applyAnthropicProviderOverrides(provider, input);
  187. expect(output).toEqual(snapshot);
  188. });
  189. it("should not change request when preference is undefined", () => {
  190. const provider = {
  191. providerType: "claude",
  192. anthropicThinkingBudgetPreference: undefined,
  193. };
  194. const input: Record<string, unknown> = {
  195. model: "claude-3-opus-20240229",
  196. messages: [],
  197. thinking: { type: "enabled", budget_tokens: 5000 },
  198. };
  199. const snapshot = structuredClone(input);
  200. const output = applyAnthropicProviderOverrides(provider, input);
  201. expect(output).toEqual(snapshot);
  202. });
  203. it("should set thinking.budget_tokens and thinking.type when preference is valid '10240'", () => {
  204. const provider = {
  205. providerType: "claude",
  206. anthropicThinkingBudgetPreference: "10240",
  207. };
  208. const input: Record<string, unknown> = {
  209. model: "claude-3-opus-20240229",
  210. messages: [],
  211. max_tokens: 32000,
  212. };
  213. const output = applyAnthropicProviderOverrides(provider, input);
  214. expect(output.thinking).toEqual({
  215. type: "enabled",
  216. budget_tokens: 10240,
  217. });
  218. });
  219. it("should preserve existing thinking properties not overridden", () => {
  220. const provider = {
  221. providerType: "claude",
  222. anthropicThinkingBudgetPreference: "8000",
  223. };
  224. const input: Record<string, unknown> = {
  225. model: "claude-3-opus-20240229",
  226. messages: [],
  227. max_tokens: 32000,
  228. thinking: {
  229. type: "disabled",
  230. budget_tokens: 2000,
  231. custom_field: "preserve_me",
  232. },
  233. };
  234. const output = applyAnthropicProviderOverrides(provider, input);
  235. const thinking = output.thinking as Record<string, unknown>;
  236. expect(thinking.type).toBe("enabled");
  237. expect(thinking.budget_tokens).toBe(8000);
  238. expect(thinking.custom_field).toBe("preserve_me");
  239. });
  240. it("should create thinking object if not present", () => {
  241. const provider = {
  242. providerType: "claude",
  243. anthropicThinkingBudgetPreference: "5000",
  244. };
  245. const input: Record<string, unknown> = {
  246. model: "claude-3-opus-20240229",
  247. messages: [],
  248. max_tokens: 10000,
  249. };
  250. const output = applyAnthropicProviderOverrides(provider, input);
  251. expect(output.thinking).toBeDefined();
  252. const thinking = output.thinking as Record<string, unknown>;
  253. expect(thinking.type).toBe("enabled");
  254. expect(thinking.budget_tokens).toBe(5000);
  255. });
  256. it("should handle non-object thinking value by replacing it", () => {
  257. const provider = {
  258. providerType: "claude",
  259. anthropicThinkingBudgetPreference: "6000",
  260. };
  261. const input: Record<string, unknown> = {
  262. model: "claude-3-opus-20240229",
  263. messages: [],
  264. max_tokens: 32000,
  265. thinking: "invalid_string_value",
  266. };
  267. const output = applyAnthropicProviderOverrides(provider, input);
  268. expect(output.thinking).toEqual({
  269. type: "enabled",
  270. budget_tokens: 6000,
  271. });
  272. });
  273. });
  274. describe("Clamping logic", () => {
  275. it("should clamp budget_tokens to max_tokens - 1 when budget_tokens >= max_tokens (overridden max_tokens)", () => {
  276. const provider = {
  277. providerType: "claude",
  278. anthropicMaxTokensPreference: "10000",
  279. anthropicThinkingBudgetPreference: "15000",
  280. };
  281. const input: Record<string, unknown> = {
  282. model: "claude-3-opus-20240229",
  283. messages: [],
  284. };
  285. const output = applyAnthropicProviderOverrides(provider, input);
  286. expect(output.max_tokens).toBe(10000);
  287. const thinking = output.thinking as Record<string, unknown>;
  288. expect(thinking.budget_tokens).toBe(9999);
  289. });
  290. it("should clamp budget_tokens to max_tokens - 1 when budget_tokens >= max_tokens (request-provided max_tokens)", () => {
  291. const provider = {
  292. providerType: "claude",
  293. anthropicThinkingBudgetPreference: "20000",
  294. };
  295. const input: Record<string, unknown> = {
  296. model: "claude-3-opus-20240229",
  297. messages: [],
  298. max_tokens: 16000,
  299. };
  300. const output = applyAnthropicProviderOverrides(provider, input);
  301. const thinking = output.thinking as Record<string, unknown>;
  302. expect(thinking.budget_tokens).toBe(15999);
  303. });
  304. it("should clamp budget_tokens when exactly equal to max_tokens", () => {
  305. const provider = {
  306. providerType: "claude",
  307. anthropicMaxTokensPreference: "8000",
  308. anthropicThinkingBudgetPreference: "8000",
  309. };
  310. const input: Record<string, unknown> = {
  311. model: "claude-3-opus-20240229",
  312. messages: [],
  313. };
  314. const output = applyAnthropicProviderOverrides(provider, input);
  315. expect(output.max_tokens).toBe(8000);
  316. const thinking = output.thinking as Record<string, unknown>;
  317. expect(thinking.budget_tokens).toBe(7999);
  318. });
  319. it("should not clamp when budget_tokens < max_tokens", () => {
  320. const provider = {
  321. providerType: "claude",
  322. anthropicMaxTokensPreference: "32000",
  323. anthropicThinkingBudgetPreference: "10000",
  324. };
  325. const input: Record<string, unknown> = {
  326. model: "claude-3-opus-20240229",
  327. messages: [],
  328. };
  329. const output = applyAnthropicProviderOverrides(provider, input);
  330. expect(output.max_tokens).toBe(32000);
  331. const thinking = output.thinking as Record<string, unknown>;
  332. expect(thinking.budget_tokens).toBe(10000);
  333. });
  334. it("should not clamp when max_tokens is not set", () => {
  335. const provider = {
  336. providerType: "claude",
  337. anthropicThinkingBudgetPreference: "50000",
  338. };
  339. const input: Record<string, unknown> = {
  340. model: "claude-3-opus-20240229",
  341. messages: [],
  342. };
  343. const output = applyAnthropicProviderOverrides(provider, input);
  344. const thinking = output.thinking as Record<string, unknown>;
  345. expect(thinking.budget_tokens).toBe(50000);
  346. });
  347. it("should skip thinking override when clamped budget_tokens would be below 1024 (API minimum)", () => {
  348. const provider = {
  349. providerType: "claude",
  350. anthropicMaxTokensPreference: "500",
  351. anthropicThinkingBudgetPreference: "10000",
  352. };
  353. const input: Record<string, unknown> = {
  354. model: "claude-3-opus-20240229",
  355. messages: [],
  356. };
  357. const output = applyAnthropicProviderOverrides(provider, input);
  358. expect(output.max_tokens).toBe(500);
  359. expect(output.thinking).toBeUndefined();
  360. });
  361. it("should skip thinking override when budget_tokens preference itself is below 1024", () => {
  362. const provider = {
  363. providerType: "claude",
  364. anthropicThinkingBudgetPreference: "500",
  365. };
  366. const input: Record<string, unknown> = {
  367. model: "claude-3-opus-20240229",
  368. messages: [],
  369. max_tokens: 32000,
  370. };
  371. const output = applyAnthropicProviderOverrides(provider, input);
  372. expect(output.thinking).toBeUndefined();
  373. });
  374. it("should skip thinking override when max_tokens is exactly 1024 (clamped would be 1023)", () => {
  375. const provider = {
  376. providerType: "claude",
  377. anthropicMaxTokensPreference: "1024",
  378. anthropicThinkingBudgetPreference: "2000",
  379. };
  380. const input: Record<string, unknown> = {
  381. model: "claude-3-opus-20240229",
  382. messages: [],
  383. };
  384. const output = applyAnthropicProviderOverrides(provider, input);
  385. expect(output.max_tokens).toBe(1024);
  386. expect(output.thinking).toBeUndefined();
  387. });
  388. it("should apply thinking override when clamped budget_tokens is exactly 1024", () => {
  389. const provider = {
  390. providerType: "claude",
  391. anthropicMaxTokensPreference: "1025",
  392. anthropicThinkingBudgetPreference: "2000",
  393. };
  394. const input: Record<string, unknown> = {
  395. model: "claude-3-opus-20240229",
  396. messages: [],
  397. };
  398. const output = applyAnthropicProviderOverrides(provider, input);
  399. expect(output.max_tokens).toBe(1025);
  400. const thinking = output.thinking as Record<string, unknown>;
  401. expect(thinking.budget_tokens).toBe(1024);
  402. expect(thinking.type).toBe("enabled");
  403. });
  404. it("should apply thinking override when budget_tokens is exactly 1024 and no clamping needed", () => {
  405. const provider = {
  406. providerType: "claude",
  407. anthropicThinkingBudgetPreference: "1024",
  408. };
  409. const input: Record<string, unknown> = {
  410. model: "claude-3-opus-20240229",
  411. messages: [],
  412. max_tokens: 32000,
  413. };
  414. const output = applyAnthropicProviderOverrides(provider, input);
  415. const thinking = output.thinking as Record<string, unknown>;
  416. expect(thinking.budget_tokens).toBe(1024);
  417. expect(thinking.type).toBe("enabled");
  418. });
  419. });
  420. describe("Audit function", () => {
  421. it("should return null audit when provider type is not claude/claude-auth", () => {
  422. const provider = {
  423. id: 123,
  424. name: "codex-provider",
  425. providerType: "codex",
  426. anthropicMaxTokensPreference: "32000",
  427. anthropicThinkingBudgetPreference: "10240",
  428. };
  429. const input: Record<string, unknown> = {
  430. model: "gpt-4",
  431. messages: [],
  432. };
  433. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  434. expect(result.request).toBe(input);
  435. expect(result.audit).toBeNull();
  436. });
  437. it("should return null audit when all preferences are inherit/null/undefined", () => {
  438. const provider = {
  439. providerType: "claude",
  440. anthropicMaxTokensPreference: "inherit",
  441. anthropicThinkingBudgetPreference: null,
  442. };
  443. const input: Record<string, unknown> = {
  444. model: "claude-3-opus-20240229",
  445. messages: [],
  446. max_tokens: 8000,
  447. };
  448. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  449. expect(result.request).toBe(input);
  450. expect(result.audit).toBeNull();
  451. });
  452. it("should return null audit when preferences are invalid numeric strings", () => {
  453. const provider = {
  454. providerType: "claude",
  455. anthropicMaxTokensPreference: "invalid",
  456. anthropicThinkingBudgetPreference: "not_a_number",
  457. };
  458. const input: Record<string, unknown> = {
  459. model: "claude-3-opus-20240229",
  460. messages: [],
  461. max_tokens: 8000,
  462. };
  463. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  464. expect(result.request).toBe(input);
  465. expect(result.audit).toBeNull();
  466. });
  467. it("should return audit with hit=true when max_tokens override is applied", () => {
  468. const provider = {
  469. id: 1,
  470. name: "claude-provider",
  471. providerType: "claude",
  472. anthropicMaxTokensPreference: "32000",
  473. };
  474. const input: Record<string, unknown> = {
  475. model: "claude-3-opus-20240229",
  476. messages: [],
  477. max_tokens: 8000,
  478. };
  479. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  480. expect(result.audit?.hit).toBe(true);
  481. expect(result.audit?.providerId).toBe(1);
  482. expect(result.audit?.providerName).toBe("claude-provider");
  483. });
  484. it("should return audit with hit=true when thinking override is applied", () => {
  485. const provider = {
  486. id: 2,
  487. name: "anthropic-direct",
  488. providerType: "claude-auth",
  489. anthropicThinkingBudgetPreference: "10240",
  490. };
  491. const input: Record<string, unknown> = {
  492. model: "claude-3-opus-20240229",
  493. messages: [],
  494. max_tokens: 32000,
  495. };
  496. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  497. expect(result.audit?.hit).toBe(true);
  498. expect(result.audit?.providerId).toBe(2);
  499. expect(result.audit?.providerName).toBe("anthropic-direct");
  500. });
  501. it("should track before/after values correctly for max_tokens", () => {
  502. const provider = {
  503. id: 1,
  504. name: "test-provider",
  505. providerType: "claude",
  506. anthropicMaxTokensPreference: "32000",
  507. };
  508. const input: Record<string, unknown> = {
  509. model: "claude-3-opus-20240229",
  510. messages: [],
  511. max_tokens: 8000,
  512. };
  513. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  514. const maxTokensChange = result.audit?.changes.find((c) => c.path === "max_tokens");
  515. expect(maxTokensChange?.before).toBe(8000);
  516. expect(maxTokensChange?.after).toBe(32000);
  517. expect(maxTokensChange?.changed).toBe(true);
  518. });
  519. it("should track before/after values correctly for thinking fields", () => {
  520. const provider = {
  521. id: 1,
  522. name: "test-provider",
  523. providerType: "claude",
  524. anthropicThinkingBudgetPreference: "10000",
  525. };
  526. const input: Record<string, unknown> = {
  527. model: "claude-3-opus-20240229",
  528. messages: [],
  529. max_tokens: 32000,
  530. thinking: { type: "disabled", budget_tokens: 5000 },
  531. };
  532. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  533. const typeChange = result.audit?.changes.find((c) => c.path === "thinking.type");
  534. expect(typeChange?.before).toBe("disabled");
  535. expect(typeChange?.after).toBe("enabled");
  536. expect(typeChange?.changed).toBe(true);
  537. const budgetChange = result.audit?.changes.find((c) => c.path === "thinking.budget_tokens");
  538. expect(budgetChange?.before).toBe(5000);
  539. expect(budgetChange?.after).toBe(10000);
  540. expect(budgetChange?.changed).toBe(true);
  541. });
  542. it("should set changed=false when override value equals existing value", () => {
  543. const provider = {
  544. id: 1,
  545. name: "test-provider",
  546. providerType: "claude",
  547. anthropicMaxTokensPreference: "8000",
  548. };
  549. const input: Record<string, unknown> = {
  550. model: "claude-3-opus-20240229",
  551. messages: [],
  552. max_tokens: 8000,
  553. };
  554. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  555. const maxTokensChange = result.audit?.changes.find((c) => c.path === "max_tokens");
  556. expect(maxTokensChange?.before).toBe(8000);
  557. expect(maxTokensChange?.after).toBe(8000);
  558. expect(maxTokensChange?.changed).toBe(false);
  559. });
  560. it("should set audit.changed=true only when at least one value actually changed", () => {
  561. const provider = {
  562. id: 1,
  563. name: "test-provider",
  564. providerType: "claude",
  565. anthropicMaxTokensPreference: "32000",
  566. anthropicThinkingBudgetPreference: "10240",
  567. };
  568. const input: Record<string, unknown> = {
  569. model: "claude-3-opus-20240229",
  570. messages: [],
  571. max_tokens: 8000,
  572. };
  573. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  574. expect(result.audit?.changed).toBe(true);
  575. });
  576. it("should set audit.changed=false when no values actually changed", () => {
  577. const provider = {
  578. id: 1,
  579. name: "test-provider",
  580. providerType: "claude",
  581. anthropicMaxTokensPreference: "8000",
  582. };
  583. const input: Record<string, unknown> = {
  584. model: "claude-3-opus-20240229",
  585. messages: [],
  586. max_tokens: 8000,
  587. };
  588. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  589. expect(result.audit?.hit).toBe(true);
  590. expect(result.audit?.changed).toBe(false);
  591. });
  592. it("should include correct metadata in audit", () => {
  593. const provider = {
  594. id: 42,
  595. name: "my-claude-provider",
  596. providerType: "claude",
  597. anthropicMaxTokensPreference: "16000",
  598. };
  599. const input: Record<string, unknown> = {
  600. model: "claude-3-opus-20240229",
  601. messages: [],
  602. };
  603. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  604. expect(result.audit?.type).toBe("provider_parameter_override");
  605. expect(result.audit?.scope).toBe("provider");
  606. expect(result.audit?.providerId).toBe(42);
  607. expect(result.audit?.providerName).toBe("my-claude-provider");
  608. expect(result.audit?.providerType).toBe("claude");
  609. });
  610. it("should handle missing provider id and name gracefully", () => {
  611. const provider = {
  612. providerType: "claude",
  613. anthropicMaxTokensPreference: "16000",
  614. };
  615. const input: Record<string, unknown> = {
  616. model: "claude-3-opus-20240229",
  617. messages: [],
  618. };
  619. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  620. expect(result.audit?.providerId).toBeNull();
  621. expect(result.audit?.providerName).toBeNull();
  622. });
  623. it("should track null before values when fields do not exist", () => {
  624. const provider = {
  625. id: 1,
  626. name: "test",
  627. providerType: "claude",
  628. anthropicMaxTokensPreference: "32000",
  629. anthropicThinkingBudgetPreference: "10000",
  630. };
  631. const input: Record<string, unknown> = {
  632. model: "claude-3-opus-20240229",
  633. messages: [],
  634. };
  635. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  636. const maxTokensChange = result.audit?.changes.find((c) => c.path === "max_tokens");
  637. expect(maxTokensChange?.before).toBeNull();
  638. expect(maxTokensChange?.after).toBe(32000);
  639. const typeChange = result.audit?.changes.find((c) => c.path === "thinking.type");
  640. expect(typeChange?.before).toBeNull();
  641. expect(typeChange?.after).toBe("enabled");
  642. const budgetChange = result.audit?.changes.find((c) => c.path === "thinking.budget_tokens");
  643. expect(budgetChange?.before).toBeNull();
  644. expect(budgetChange?.after).toBe(10000);
  645. });
  646. });
  647. describe("Adaptive thinking mode", () => {
  648. it("should apply adaptive thinking for matching model (all models mode)", () => {
  649. const provider = {
  650. providerType: "claude",
  651. anthropicAdaptiveThinking: {
  652. effort: "high" as const,
  653. modelMatchMode: "all" as const,
  654. models: [],
  655. },
  656. };
  657. const input: Record<string, unknown> = {
  658. model: "claude-opus-4-6",
  659. messages: [],
  660. max_tokens: 8000,
  661. };
  662. const output = applyAnthropicProviderOverrides(provider, input);
  663. expect(output.thinking).toEqual({ type: "adaptive" });
  664. expect(output.output_config).toEqual({ effort: "high" });
  665. });
  666. it("should apply adaptive thinking for matching model (specific models mode)", () => {
  667. const provider = {
  668. providerType: "claude",
  669. anthropicAdaptiveThinking: {
  670. effort: "max" as const,
  671. modelMatchMode: "specific" as const,
  672. models: ["claude-opus-4-6"],
  673. },
  674. };
  675. const input: Record<string, unknown> = {
  676. model: "claude-opus-4-6",
  677. messages: [],
  678. };
  679. const output = applyAnthropicProviderOverrides(provider, input);
  680. expect(output.thinking).toEqual({ type: "adaptive" });
  681. expect(output.output_config).toEqual({ effort: "max" });
  682. });
  683. it("should passthrough for non-matching model (specific models mode)", () => {
  684. const provider = {
  685. providerType: "claude",
  686. anthropicAdaptiveThinking: {
  687. effort: "high" as const,
  688. modelMatchMode: "specific" as const,
  689. models: ["claude-opus-4-6"],
  690. },
  691. };
  692. const input: Record<string, unknown> = {
  693. model: "claude-sonnet-4-5",
  694. messages: [],
  695. max_tokens: 8000,
  696. thinking: { type: "enabled", budget_tokens: 5000 },
  697. };
  698. const snapshot = structuredClone(input);
  699. const output = applyAnthropicProviderOverrides(provider, input);
  700. expect(output).toEqual(snapshot);
  701. });
  702. it("should preserve existing output_config properties", () => {
  703. const provider = {
  704. providerType: "claude",
  705. anthropicAdaptiveThinking: {
  706. effort: "medium" as const,
  707. modelMatchMode: "all" as const,
  708. models: [],
  709. },
  710. };
  711. const input: Record<string, unknown> = {
  712. model: "claude-opus-4-6",
  713. messages: [],
  714. output_config: { some_other_field: "preserve" },
  715. };
  716. const output = applyAnthropicProviderOverrides(provider, input);
  717. const outputConfig = output.output_config as Record<string, unknown>;
  718. expect(outputConfig.effort).toBe("medium");
  719. expect(outputConfig.some_other_field).toBe("preserve");
  720. });
  721. it("should apply adaptive with effort 'low'", () => {
  722. const provider = {
  723. providerType: "claude",
  724. anthropicAdaptiveThinking: {
  725. effort: "low" as const,
  726. modelMatchMode: "all" as const,
  727. models: [],
  728. },
  729. };
  730. const input: Record<string, unknown> = {
  731. model: "claude-opus-4-6",
  732. messages: [],
  733. };
  734. const output = applyAnthropicProviderOverrides(provider, input);
  735. expect(output.output_config).toEqual({ effort: "low" });
  736. });
  737. it("should remove budget_tokens from existing thinking when applying adaptive", () => {
  738. const provider = {
  739. providerType: "claude",
  740. anthropicAdaptiveThinking: {
  741. effort: "high" as const,
  742. modelMatchMode: "all" as const,
  743. models: [],
  744. },
  745. };
  746. const input: Record<string, unknown> = {
  747. model: "claude-opus-4-6",
  748. messages: [],
  749. thinking: { type: "enabled", budget_tokens: 10240 },
  750. };
  751. const output = applyAnthropicProviderOverrides(provider, input);
  752. const thinking = output.thinking as Record<string, unknown>;
  753. expect(thinking.type).toBe("adaptive");
  754. expect(thinking.budget_tokens).toBeUndefined();
  755. });
  756. it("should passthrough when adaptive config is null (defensive)", () => {
  757. const provider = {
  758. providerType: "claude",
  759. anthropicAdaptiveThinking: null,
  760. };
  761. const input: Record<string, unknown> = {
  762. model: "claude-opus-4-6",
  763. messages: [],
  764. max_tokens: 8000,
  765. };
  766. const snapshot = structuredClone(input);
  767. const output = applyAnthropicProviderOverrides(provider, input);
  768. expect(output).toEqual(snapshot);
  769. });
  770. it("should apply adaptive + max_tokens override together", () => {
  771. const provider = {
  772. providerType: "claude",
  773. anthropicMaxTokensPreference: "32000",
  774. anthropicAdaptiveThinking: {
  775. effort: "high" as const,
  776. modelMatchMode: "all" as const,
  777. models: [],
  778. },
  779. };
  780. const input: Record<string, unknown> = {
  781. model: "claude-opus-4-6",
  782. messages: [],
  783. max_tokens: 8000,
  784. };
  785. const output = applyAnthropicProviderOverrides(provider, input);
  786. expect(output.max_tokens).toBe(32000);
  787. expect(output.thinking).toEqual({ type: "adaptive" });
  788. expect(output.output_config).toEqual({ effort: "high" });
  789. });
  790. it("should match model prefix (claude-opus-4-6 matches claude-opus-4-6-20250514)", () => {
  791. const provider = {
  792. providerType: "claude",
  793. anthropicAdaptiveThinking: {
  794. effort: "high" as const,
  795. modelMatchMode: "specific" as const,
  796. models: ["claude-opus-4-6"],
  797. },
  798. };
  799. const input: Record<string, unknown> = {
  800. model: "claude-opus-4-6-20250514",
  801. messages: [],
  802. };
  803. const output = applyAnthropicProviderOverrides(provider, input);
  804. expect(output.thinking).toEqual({ type: "adaptive" });
  805. expect(output.output_config).toEqual({ effort: "high" });
  806. });
  807. it("should track output_config.effort in audit for adaptive mode", () => {
  808. const provider = {
  809. id: 1,
  810. name: "adaptive-provider",
  811. providerType: "claude",
  812. anthropicAdaptiveThinking: {
  813. effort: "high" as const,
  814. modelMatchMode: "all" as const,
  815. models: [],
  816. },
  817. };
  818. const input: Record<string, unknown> = {
  819. model: "claude-opus-4-6",
  820. messages: [],
  821. };
  822. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  823. expect(result.audit?.hit).toBe(true);
  824. expect(result.audit?.changed).toBe(true);
  825. const effortChange = result.audit?.changes.find((c) => c.path === "output_config.effort");
  826. expect(effortChange?.before).toBeNull();
  827. expect(effortChange?.after).toBe("high");
  828. expect(effortChange?.changed).toBe(true);
  829. const thinkingTypeChange = result.audit?.changes.find((c) => c.path === "thinking.type");
  830. expect(thinkingTypeChange?.after).toBe("adaptive");
  831. });
  832. });
  833. describe("Adaptive + Budget coexistence", () => {
  834. it("should apply adaptive when model matches, ignoring budget override", () => {
  835. const provider = {
  836. providerType: "claude",
  837. anthropicThinkingBudgetPreference: "10240",
  838. anthropicAdaptiveThinking: {
  839. effort: "high" as const,
  840. modelMatchMode: "specific" as const,
  841. models: ["claude-opus-4-6"],
  842. },
  843. };
  844. const input: Record<string, unknown> = {
  845. model: "claude-opus-4-6",
  846. messages: [],
  847. max_tokens: 32000,
  848. };
  849. const output = applyAnthropicProviderOverrides(provider, input);
  850. expect(output.thinking).toEqual({ type: "adaptive" });
  851. expect(output.output_config).toEqual({ effort: "high" });
  852. // Budget should NOT be applied when adaptive matches
  853. const thinking = output.thinking as Record<string, unknown>;
  854. expect(thinking.budget_tokens).toBeUndefined();
  855. });
  856. it("should fallback to budget when model does not match adaptive config", () => {
  857. const provider = {
  858. providerType: "claude",
  859. anthropicThinkingBudgetPreference: "10240",
  860. anthropicAdaptiveThinking: {
  861. effort: "high" as const,
  862. modelMatchMode: "specific" as const,
  863. models: ["claude-opus-4-6"],
  864. },
  865. };
  866. const input: Record<string, unknown> = {
  867. model: "claude-sonnet-4-5",
  868. messages: [],
  869. max_tokens: 32000,
  870. };
  871. const output = applyAnthropicProviderOverrides(provider, input);
  872. const thinking = output.thinking as Record<string, unknown>;
  873. expect(thinking.type).toBe("enabled");
  874. expect(thinking.budget_tokens).toBe(10240);
  875. expect(output.output_config).toBeUndefined();
  876. });
  877. it("should passthrough when model does not match adaptive and budget is inherit", () => {
  878. const provider = {
  879. providerType: "claude",
  880. anthropicThinkingBudgetPreference: "inherit",
  881. anthropicAdaptiveThinking: {
  882. effort: "high" as const,
  883. modelMatchMode: "specific" as const,
  884. models: ["claude-opus-4-6"],
  885. },
  886. };
  887. const input: Record<string, unknown> = {
  888. model: "claude-sonnet-4-5",
  889. messages: [],
  890. max_tokens: 32000,
  891. thinking: { type: "enabled", budget_tokens: 5000 },
  892. };
  893. const snapshot = structuredClone(input);
  894. const output = applyAnthropicProviderOverrides(provider, input);
  895. expect(output).toEqual(snapshot);
  896. });
  897. it("should always apply adaptive when modelMatchMode=all, regardless of budget", () => {
  898. const provider = {
  899. providerType: "claude",
  900. anthropicThinkingBudgetPreference: "10240",
  901. anthropicAdaptiveThinking: {
  902. effort: "max" as const,
  903. modelMatchMode: "all" as const,
  904. models: [],
  905. },
  906. };
  907. const input: Record<string, unknown> = {
  908. model: "claude-sonnet-4-5",
  909. messages: [],
  910. max_tokens: 32000,
  911. };
  912. const output = applyAnthropicProviderOverrides(provider, input);
  913. expect(output.thinking).toEqual({ type: "adaptive" });
  914. expect(output.output_config).toEqual({ effort: "max" });
  915. });
  916. it("should produce correct audit trail when adaptive matches (coexistence)", () => {
  917. const provider = {
  918. id: 10,
  919. name: "coexist-provider",
  920. providerType: "claude",
  921. anthropicThinkingBudgetPreference: "10240",
  922. anthropicAdaptiveThinking: {
  923. effort: "high" as const,
  924. modelMatchMode: "specific" as const,
  925. models: ["claude-opus-4-6"],
  926. },
  927. };
  928. const input: Record<string, unknown> = {
  929. model: "claude-opus-4-6",
  930. messages: [],
  931. max_tokens: 32000,
  932. };
  933. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  934. expect(result.audit?.hit).toBe(true);
  935. expect(result.audit?.changed).toBe(true);
  936. const effortChange = result.audit?.changes.find((c) => c.path === "output_config.effort");
  937. expect(effortChange?.after).toBe("high");
  938. expect(effortChange?.changed).toBe(true);
  939. const thinkingTypeChange = result.audit?.changes.find((c) => c.path === "thinking.type");
  940. expect(thinkingTypeChange?.after).toBe("adaptive");
  941. });
  942. it("should produce correct audit trail when falling back to budget (coexistence)", () => {
  943. const provider = {
  944. id: 10,
  945. name: "coexist-provider",
  946. providerType: "claude",
  947. anthropicThinkingBudgetPreference: "10240",
  948. anthropicAdaptiveThinking: {
  949. effort: "high" as const,
  950. modelMatchMode: "specific" as const,
  951. models: ["claude-opus-4-6"],
  952. },
  953. };
  954. const input: Record<string, unknown> = {
  955. model: "claude-sonnet-4-5",
  956. messages: [],
  957. max_tokens: 32000,
  958. };
  959. const result = applyAnthropicProviderOverridesWithAudit(provider, input);
  960. expect(result.audit?.hit).toBe(true);
  961. expect(result.audit?.changed).toBe(true);
  962. const thinkingTypeChange = result.audit?.changes.find((c) => c.path === "thinking.type");
  963. expect(thinkingTypeChange?.after).toBe("enabled");
  964. const budgetChange = result.audit?.changes.find((c) => c.path === "thinking.budget_tokens");
  965. expect(budgetChange?.after).toBe(10240);
  966. // output_config.effort should NOT be set for budget fallback
  967. const effortChange = result.audit?.changes.find((c) => c.path === "output_config.effort");
  968. expect(effortChange?.changed).toBe(false);
  969. });
  970. });
  971. });