followup-after-completion.ts 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135
  1. import { runStreamCase, StreamEvent } from "../lib/stream-harness"
  2. const FIRST_PROMPT = `What is 1+1? Reply with only "2".`
  3. const FOLLOWUP_PROMPT = `Different question now: what is 3+3? Reply with only "6".`
  4. function parseEventContent(text: string | undefined): string {
  5. return typeof text === "string" ? text : ""
  6. }
  7. function validateFollowupResult(text: string): void {
  8. if (text.trim().length === 0) {
  9. throw new Error("follow-up produced an empty result")
  10. }
  11. }
  12. async function main() {
  13. const startRequestId = `start-${Date.now()}`
  14. const followupRequestId = `message-${Date.now()}`
  15. const shutdownRequestId = `shutdown-${Date.now()}`
  16. let initSeen = false
  17. let sentFollowup = false
  18. let sentShutdown = false
  19. let firstResult = ""
  20. let followupResult = ""
  21. let followupDoneCode: string | undefined
  22. let sawFollowupUserTurn = false
  23. let sawMisroutedToolResult = false
  24. await runStreamCase({
  25. onEvent(event: StreamEvent, context) {
  26. if (event.type === "system" && event.subtype === "init" && !initSeen) {
  27. initSeen = true
  28. context.sendCommand({
  29. command: "start",
  30. requestId: startRequestId,
  31. prompt: FIRST_PROMPT,
  32. })
  33. return
  34. }
  35. if (event.type === "control" && event.subtype === "error") {
  36. throw new Error(
  37. `received control error for requestId=${event.requestId ?? "unknown"} command=${event.command ?? "unknown"} code=${event.code ?? "unknown"} content=${event.content ?? ""}`,
  38. )
  39. }
  40. if (event.type !== "result" || event.done !== true) {
  41. if (
  42. event.type === "control" &&
  43. event.requestId === followupRequestId &&
  44. event.command === "message" &&
  45. event.subtype === "done"
  46. ) {
  47. followupDoneCode = event.code
  48. return
  49. }
  50. if (
  51. event.type === "tool_result" &&
  52. event.requestId === followupRequestId &&
  53. typeof event.content === "string" &&
  54. event.content.includes("<user_message>")
  55. ) {
  56. sawMisroutedToolResult = true
  57. return
  58. }
  59. if (event.type === "user" && event.requestId === followupRequestId) {
  60. sawFollowupUserTurn = typeof event.content === "string" && event.content.includes("3+3")
  61. return
  62. }
  63. return
  64. }
  65. if (event.requestId === startRequestId) {
  66. firstResult = parseEventContent(event.content)
  67. if (!/\b2\b/.test(firstResult)) {
  68. throw new Error(`first result did not answer first prompt; result="${firstResult}"`)
  69. }
  70. if (!sentFollowup) {
  71. context.sendCommand({
  72. command: "message",
  73. requestId: followupRequestId,
  74. prompt: FOLLOWUP_PROMPT,
  75. })
  76. sentFollowup = true
  77. }
  78. return
  79. }
  80. if (event.requestId !== followupRequestId) {
  81. return
  82. }
  83. followupResult = parseEventContent(event.content)
  84. validateFollowupResult(followupResult)
  85. if (followupDoneCode !== "responded") {
  86. throw new Error(
  87. `follow-up message was not routed as ask response; code="${followupDoneCode ?? "none"}"`,
  88. )
  89. }
  90. if (!sawFollowupUserTurn) {
  91. throw new Error("follow-up did not appear as a normal user turn in stream output")
  92. }
  93. if (sawMisroutedToolResult) {
  94. throw new Error("follow-up message was misrouted into tool_result (<user_message>), old bug reproduced")
  95. }
  96. console.log(`[PASS] first result="${firstResult}"`)
  97. console.log(`[PASS] follow-up result="${followupResult}"`)
  98. if (!sentShutdown) {
  99. context.sendCommand({
  100. command: "shutdown",
  101. requestId: shutdownRequestId,
  102. })
  103. sentShutdown = true
  104. }
  105. },
  106. onTimeoutMessage() {
  107. return `timed out waiting for completion (initSeen=${initSeen}, sentFollowup=${sentFollowup}, firstResult=${Boolean(firstResult)}, followupResult=${Boolean(followupResult)})`
  108. },
  109. })
  110. }
  111. main().catch((error) => {
  112. console.error(`[FAIL] ${error instanceof Error ? error.message : String(error)}`)
  113. process.exit(1)
  114. })