mixed-command-ordering.ts 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148
  1. import { runStreamCase, StreamEvent } from "../lib/stream-harness"
  2. const START_PROMPT =
  3. 'Run exactly this command and do not summarize until it finishes: sleep 8 && echo "done". After it finishes, reply with exactly "done".'
  4. async function main() {
  5. const startRequestId = `start-${Date.now()}`
  6. const pingARequestId = `ping-a-${Date.now()}`
  7. const messageRequestId = `message-${Date.now()}`
  8. const pingBRequestId = `ping-b-${Date.now()}`
  9. const shutdownRequestId = `shutdown-${Date.now()}`
  10. let initSeen = false
  11. let sentInterleavedCommands = false
  12. let sentShutdown = false
  13. const eventOrderByRequestId = new Map<string, string[]>()
  14. let messageDoneCode: string | undefined
  15. let messageQueueEnqueuedSeen = false
  16. let messageResultSeen = false
  17. function recordControlEvent(event: StreamEvent): void {
  18. if (!event.requestId || event.type !== "control" || !event.subtype) {
  19. return
  20. }
  21. const existing = eventOrderByRequestId.get(event.requestId) ?? []
  22. existing.push(event.subtype)
  23. eventOrderByRequestId.set(event.requestId, existing)
  24. }
  25. await runStreamCase({
  26. onEvent(event: StreamEvent, context) {
  27. if (event.type === "system" && event.subtype === "init" && !initSeen) {
  28. initSeen = true
  29. context.sendCommand({
  30. command: "start",
  31. requestId: startRequestId,
  32. prompt: START_PROMPT,
  33. })
  34. return
  35. }
  36. recordControlEvent(event)
  37. if (event.type === "control" && event.subtype === "error") {
  38. throw new Error(
  39. `received control error for requestId=${event.requestId ?? "unknown"} command=${event.command ?? "unknown"} code=${event.code ?? "unknown"} content=${event.content ?? ""}`,
  40. )
  41. }
  42. if (
  43. !sentInterleavedCommands &&
  44. event.type === "control" &&
  45. event.subtype === "ack" &&
  46. event.command === "start" &&
  47. event.requestId === startRequestId
  48. ) {
  49. context.sendCommand({
  50. command: "ping",
  51. requestId: pingARequestId,
  52. })
  53. context.sendCommand({
  54. command: "message",
  55. requestId: messageRequestId,
  56. prompt: 'When this queued message is processed, reply with only "INTERLEAVED".',
  57. })
  58. context.sendCommand({
  59. command: "ping",
  60. requestId: pingBRequestId,
  61. })
  62. sentInterleavedCommands = true
  63. return
  64. }
  65. if (
  66. event.type === "control" &&
  67. event.subtype === "done" &&
  68. event.command === "message" &&
  69. event.requestId === messageRequestId
  70. ) {
  71. messageDoneCode = event.code
  72. return
  73. }
  74. if (
  75. event.type === "queue" &&
  76. event.subtype === "enqueued" &&
  77. event.requestId === startRequestId &&
  78. event.queueDepth === 1
  79. ) {
  80. messageQueueEnqueuedSeen = true
  81. return
  82. }
  83. if (event.type === "result" && event.done === true && event.requestId === messageRequestId) {
  84. messageResultSeen = true
  85. const pingAOrder = eventOrderByRequestId.get(pingARequestId) ?? []
  86. const pingBOrder = eventOrderByRequestId.get(pingBRequestId) ?? []
  87. const messageOrder = eventOrderByRequestId.get(messageRequestId) ?? []
  88. if (pingAOrder.join(",") !== "ack,done") {
  89. throw new Error(`ping A control order mismatch: ${pingAOrder.join(",") || "none"}`)
  90. }
  91. if (pingBOrder.join(",") !== "ack,done") {
  92. throw new Error(`ping B control order mismatch: ${pingBOrder.join(",") || "none"}`)
  93. }
  94. if (messageOrder.join(",") !== "ack,done") {
  95. throw new Error(`message control order mismatch: ${messageOrder.join(",") || "none"}`)
  96. }
  97. if (messageDoneCode !== "queued") {
  98. throw new Error(
  99. `expected interleaved message done code \"queued\", got \"${messageDoneCode ?? "none"}\"`,
  100. )
  101. }
  102. if (!messageQueueEnqueuedSeen) {
  103. throw new Error("expected queue enqueued event after interleaved message")
  104. }
  105. if (!sentShutdown) {
  106. context.sendCommand({
  107. command: "shutdown",
  108. requestId: shutdownRequestId,
  109. })
  110. sentShutdown = true
  111. }
  112. }
  113. },
  114. onTimeoutMessage() {
  115. return [
  116. "timed out waiting for mixed-command-ordering validation",
  117. `initSeen=${initSeen}`,
  118. `sentInterleavedCommands=${sentInterleavedCommands}`,
  119. `messageDoneCode=${messageDoneCode ?? "none"}`,
  120. `messageQueueEnqueuedSeen=${messageQueueEnqueuedSeen}`,
  121. `messageResultSeen=${messageResultSeen}`,
  122. `pingAOrder=${(eventOrderByRequestId.get(pingARequestId) ?? []).join(",") || "none"}`,
  123. `messageOrder=${(eventOrderByRequestId.get(messageRequestId) ?? []).join(",") || "none"}`,
  124. `pingBOrder=${(eventOrderByRequestId.get(pingBRequestId) ?? []).join(",") || "none"}`,
  125. ].join(" ")
  126. },
  127. })
  128. }
  129. main().catch((error) => {
  130. console.error(`[FAIL] ${error instanceof Error ? error.message : String(error)}`)
  131. process.exit(1)
  132. })