Transcript doctor II

Two model calls either way, and neither run is refused by anything. What does moving that one line cost?

import harness, lab

TESTS = "pytest"
SYSTEM = "You are a careful debugger."
PROMPT = "The tests are failing. Have a look."


class NotingShell:
    """lab.Shell, with a note of every command that really reached it."""

    def __init__(self, ws):
        self._shell, self.ran = lab.Shell(ws), []

    def run(self, command):
        self.ran.append(command)
        return self._shell.run(command)


def their_loop(model, system, messages, tools, prompt, *, max_turns=None,
               get_steering=None, get_follow_ups=None):
    """Yours from lesson 10, line for line, with the max_turns check moved: it sits below the
    model call, once the reply has been appended and announced, instead of above it. Counted
    with >= so the run still stops on the second model call, and it closes its turn and the run
    on the way out. The error_message branch has nowhere left to go, so it is gone."""
    new = []

    def record(message):
        messages.append(message)
        new.append(message)
        return {"type": "message_end", "message": message}

    def ask(get):
        return list(get()) if get else []

    yield {"type": "agent_start"}
    turn, calls, failed = 0, [], False
    pending = [harness.user_message(prompt)] + ask(get_steering)
    while pending:
        while calls or pending:
            turn += 1
            yield {"type": "turn_start"}
            for message in pending:
                yield record(message)
            pending = []
            reply = model.complete(system, harness.context_for_model(messages),
                                   harness.tool_specs(tools))
            yield record(reply)
            if max_turns is not None and turn >= max_turns:     # the moved check
                yield {"type": "turn_end", "message": reply}
                yield {"type": "agent_end", "messages": new}
                return
            failed = reply["stop_reason"] == "error"
            calls = [] if failed else harness.tool_calls(reply)
            for call in calls:
                yield {"type": "tool_execution_start", "call": call}
                result = harness.run_tool(tools, call)
                yield record(result)
                yield {"type": "tool_execution_end", "call": call, "result": result}
            yield {"type": "turn_end", "message": reply}
            if not failed:
                pending = ask(get_steering)
        if not failed:
            pending = ask(get_follow_ups)
    yield {"type": "agent_end", "messages": new}


def model():
    """A model that answers every request by asking for the tests again."""
    return lab.ScriptedModel([lab.forever(
        lab.reply(lab.text("Running the tests."), lab.call("bash", {"command": TESTS})))])


async def run(label, loop, messages):
    shell = NotingShell(lab.Workspace({"main.py": "def total(xs):\n    return sum(xs) - 1\n"}))
    calls = model()
    await lab.drive(loop(calls, SYSTEM, messages, [harness.make_bash_tool(shell)],
                         PROMPT, max_turns=2))
    view = harness.context_for_model(messages)
    c2 = next(m for m in view if m.get("tool_call_id") == "c2")
    print(f"{label}: {len(calls.calls)} model call(s); commands that reached the shell: {shell.ran}")
    print(f"   the record:                      {lab.shape(messages)}")
    print(f"   the next request carries:        {lab.shape(view)}")
    print(f"   the last line of c2's result:    {c2['content'].splitlines()[-1]!r}")


theirs, yours = [], []
await run("theirs", their_loop, theirs)
print()
await run("yours ", harness.run_agent, yours)
print()
for label, messages in (("theirs", theirs), ("yours ", yours)):
    last = messages[-1]
    print(f"{label}: the record's last message says: "
          f"{last.get('error_message') or harness.text_of(last) or '(nothing)'}")

theirs: 2 model call(s); commands that reached the shell: ['pytest']
   the record:                      U A[c1] R(c1) A[c2]
   the next request carries:        U A[c1] R(c1) A[c2] R(c2)
   the last line of c2's result:    'Tool call interrupted: no result was recorded. It may not have run, or may have run partly; check before repeating it.'

yours : 2 model call(s); commands that reached the shell: ['pytest', 'pytest']
   the record:                      U A[c1] R(c1) A[c2] R(c2) A(error)
   the next request carries:        U A[c1] R(c1) A[c2] R(c2)
   the last line of c2's result:    'Command exited with code 1'

theirs: the record's last message says: Running the tests.
yours : the record's last message says: Agent stopped after max_turns=2

Nothing is persisted, nothing is shared on purpose, and a restart clears it. What is happening?

import re

import harness, lab

SYSTEM = "You are a careful debugger."
QUESTION = "What port do we listen on?"

# The opening every conversation in their app starts from, built once when the process starts.
OPENING = [lab.user("House style: spaces, not tabs. Never edit anything under vendor/."),
           lab.say("Understood.")]


class TheirHarness(harness.Harness):
    """Yours, with one line changed: __init__ keeps the sequence it was handed instead of
    copying it into a list of its own."""

    def __init__(self, model, system, tools, messages=(), max_turns=None):
        super().__init__(model, system, tools, messages, max_turns)
        self._messages = messages               # the changed line: no list(...)


def agent(request):
    """Answers out of the request if a port is anywhere in it; otherwise asks for config.py."""
    found = re.search(r"PORT = (\d+)", request.text)
    return lab.say(f"The port is {found.group(1)}.") if found else \
        lab.reply(lab.call("read", {"path": "config.py"}))


async def conversation(kind, port):
    """One customer's conversation, on its own workspace, over the shared opening."""
    ws = lab.Workspace({"config.py": f"PORT = {port}\n"})
    model = lab.ScriptedModel([lab.forever(agent)])
    h = kind(model, SYSTEM, [harness.make_read_tool(ws)], OPENING)
    starts_with = lab.shape(h.messages)
    await lab.drive(h.prompt(QUESTION))
    return h, model, ws, starts_with


for kind, name in ((TheirHarness, "theirs"), (harness.Harness, "yours ")):
    OPENING[:] = OPENING[:2]                    # back to the two messages the app starts from
    a, _, _, _ = await conversation(kind, 9090)
    after_a = lab.shape(a.messages)
    b, model, ws, starts_with = await conversation(kind, 5432)
    print(f"{name}: after customer A's job, A's record is: {after_a}")
    print(f"        customer B opens a new conversation. It starts with: {starts_with}")
    print(f"        B's first request carried {len(model.calls[0].messages)} message(s), "
          f"and B's own workspace was read: {ws.reads}")
    print(f"        B was told: {harness.text_of(b.messages[-1])}")
    print(f"        A's record now reads: {lab.shape(a.messages)}")
    print(f"        the app's opening list now holds {len(OPENING)} message(s)")
    print()

theirs: after customer A's job, A's record is: U A U A[c1] R(c1) A
        customer B opens a new conversation. It starts with: U A U A[c1] R(c1) A
        B's first request carried 7 message(s), and B's own workspace was read: []
        B was told: The port is 9090.
        A's record now reads: U A U A[c1] R(c1) A U A
        the app's opening list now holds 8 message(s)

yours : after customer A's job, A's record is: U A U A[c1] R(c1) A
        customer B opens a new conversation. It starts with: U A
        B's first request carried 3 message(s), and B's own workspace was read: ['config.py']
        B was told: The port is 5432.
        A's record now reads: U A U A[c1] R(c1) A
        the app's opening list now holds 2 message(s)

Their agent gives up after two model calls, with the failing assertion sitting in the very result it was handed. Yours fixes the bug in four. Which idea is missing?

import harness, lab

SYSTEM = "You are a careful debugger."
PROMPT = "The test suite is red. Find out why and fix it."
BROKEN = "def total(xs):\n    return sum(xs) - 1\n"
FIXED = "def total(xs):\n    return sum(xs)\n"


def their_bash_tool(shell, max_lines=harness.MAX_LINES, max_bytes=harness.MAX_BYTES):
    """Yours, keeping its name, description, parameters and budget: only the last branch
    differs. A command that exits non-zero raises, instead of coming back as an ordinary
    result with a status line on the end."""
    tool = dict(harness.make_bash_tool(shell, max_lines, max_bytes))

    def execute(arguments):
        output, code = shell.run(harness.str_arg(arguments, "command"))
        total = len(output.splitlines())
        output, shown = harness.truncate_tail(output, max_lines, max_bytes)
        if shown < total:
            output += f"\n[Showing the last {shown} of {total} lines. Narrow the command for more.]"
        if code != 0:
            raise RuntimeError(output)          # the changed branch
        return output

    return {**tool, "execute": execute}


def agent(request):
    """Runs the tests. If the result is flagged as a tool error it stops and says the tool is
    broken. Otherwise it reads the failing assertion, writes the fix, and runs the tests again."""
    result = request.last_result
    if result is None:
        return lab.reply(lab.text("Let me run the tests."),
                         lab.call("bash", {"command": "pytest"}))
    if result["is_error"]:
        return lab.say("I cannot run the tests: the bash tool reports that it could not do its "
                       "job. Somebody should look at the tool.")
    if "assert 5 == 6" in result["content"]:
        return lab.reply(lab.text("total() is off by one."),
                         lab.call("write", {"path": "cart.py", "content": FIXED}))
    if "2992 passed" in result["content"]:
        return lab.say("Fixed: total() dropped one item. The suite is green.")
    return lab.reply(lab.call("bash", {"command": "pytest"}))


async def run(name, make_bash):
    ws = lab.Workspace({"cart.py": BROKEN})
    shell = lab.Shell(ws, tests_pass_when=lambda w: "- 1" not in w.read_text("cart.py"))
    model, messages = lab.ScriptedModel([lab.forever(agent)]), []
    await lab.drive(harness.run_agent(model, SYSTEM, messages,
                                      [make_bash(shell), harness.make_write_tool(ws)], PROMPT))
    first = messages[2]                         # the result of the first pytest call
    print(f"{name}: {len(model.calls)} model call(s), record {lab.shape(messages)}")
    print(f"        the first pytest result: is_error={first['is_error']}, "
          f"and it holds the failing assertion: {'assert 5 == 6' in first['content']}")
    print(f"        its last line: {first['content'].splitlines()[-1]!r}")
    print(f"        cart.py on disk now: {ws.read_text('cart.py').splitlines()[-1].strip()!r}")
    print(f"        the model's last word: {harness.text_of(messages[-1])}")
    return first["content"]


theirs = await run("theirs", their_bash_tool)
print()
yours = await run("yours ", harness.make_bash_tool)
print()
without_the_status_line = yours.rsplit("\n", 1)[0]
print(f"theirs is yours without its last line, character for character: "
      f"{theirs == without_the_status_line}")

theirs: 2 model call(s), record U A[c1] R(c1) A
        the first pytest result: is_error=True, and it holds the failing assertion: True
        its last line: '[Showing the last 1283 of 3000 lines. Narrow the command for more.]'
        cart.py on disk now: 'return sum(xs) - 1'
        the model's last word: I cannot run the tests: the bash tool reports that it could not do its job. Somebody should look at the tool.

yours : 4 model call(s), record U A[c1] R(c1) A[c2] R(c2) A[c3] R(c3) A
        the first pytest result: is_error=False, and it holds the failing assertion: True
        its last line: 'Command exited with code 1'
        cart.py on disk now: 'return sum(xs)'
        the model's last word: Fixed: total() dropped one item. The suite is green.

theirs is yours without its last line, character for character: True

The tab had been closed an hour earlier, while bash was starting, so nothing ran. Their repair filled the hole with The tool ran., not marked as an error. What went wrong, and where?

import harness, lab

SYSTEM = "You are a careful release engineer."
PROMPT = "Apply the pending database migration."
LATER = "Did the migration go through?"
COMFORTABLE = "The tool ran."

REFERENCE_REPAIR = harness.repair_tool_history


def their_repair(messages):
    """Yours, with two values changed in the result it makes up for a call that has none: the
    text says the tool ran, and it is not flagged as an error."""
    return [dict(m, content=COMFORTABLE, is_error=False)
            if m.get("content") == harness.INTERRUPTED else m
            for m in REFERENCE_REPAIR(messages)]


class MigrationShell:
    """Stands in for a shell: it notes every command it is given, and answers the migration."""

    def __init__(self):
        self.ran = []

    def run(self, command):
        self.ran.append(command)
        return "applying 3 migrations... done\n", 0


def agent(request):
    """Answers out of the newest tool result: a result that is flagged as an error, or that says
    no result was recorded, means it cannot confirm anything; anything else means it went through."""
    result = request.last_result
    if result is None:
        return lab.reply(lab.text("Applying the migration."),
                         lab.call("bash", {"command": "./migrate.sh"}))
    if result["is_error"] or "no result was recorded" in result["content"]:
        return lab.say("I cannot confirm it: that call was interrupted and no result was "
                       "recorded. Check the database before running it again.")
    return lab.say("Yes. The migration has been applied.")


async def run(name, repair):
    harness.repair_tool_history = repair        # the only difference between the two runs
    shell = MigrationShell()
    model = lab.ScriptedModel([lab.forever(agent)])
    h = harness.Harness(model, SYSTEM, [harness.make_bash_tool(shell)])
    # The tab is closed while bash is starting: the consumer takes five events and no more.
    events = await lab.drive(h.prompt(PROMPT), stop_after=5)
    await lab.drive(h.prompt(LATER))
    view = harness.context_for_model(list(h.messages))
    made_up = next(m for m in view if m.get("tool_call_id") == "c1")
    print(f"{name}: the screen's last event was {events[-1]['type']}; "
          f"commands that reached the shell: {shell.ran}")
    print(f"        the record:               {lab.shape(h.messages)}")
    print(f"        the next request carried: {lab.shape(view)}")
    print(f"        c1's made-up result:      {made_up['content']!r} "
          f"(is_error={made_up['is_error']})")
    print(f"        and the user was told:    {harness.text_of(h.messages[-1])}")


await run("theirs", their_repair)
print()
await run("yours ", REFERENCE_REPAIR)

theirs: the screen's last event was tool_execution_start; commands that reached the shell: []
        the record:               U A[c1] U A
        the next request carried: U A[c1] R(c1) U A
        c1's made-up result:      'The tool ran.' (is_error=False)
        and the user was told:    Yes. The migration has been applied.

yours : the screen's last event was tool_execution_start; commands that reached the shell: []
        the record:               U A[c1] U A
        the next request carried: U A[c1] R(c1) U A
        c1's made-up result:      'Tool call interrupted: no result was recorded. It may not have run, or may have run partly; check before repeating it.' (is_error=True)
        and the user was told:    I cannot confirm it: that call was interrupted and no result was recorded. Check the database before running it again.