Tell the model what went wrong

You call run_agent with that script and that tool list. The first reply asks for raed. Your run_tool from lesson 3 looks the name up in a dict of the tools it was given. What happens?

import harness, lab

ws = lab.Workspace({"config.py": "PORT = 9090\n"})

def retry(request):     # it corrects a typo it has been told about
    result = request.last_result
    if result and result["is_error"] and "raed" in result["content"]:
        return lab.reply(lab.call("read", {"path": "config.py"}))
    return lab.say("I cannot continue.")

model = lab.ScriptedModel([
    lab.reply(lab.call("raed", {"path": "config.py"})),   # a typo
    retry,
    lambda request: lab.say("The port is 9090."),
])

messages = []
reply = harness.run_agent(model, "You are helpful.", messages,
                          [harness.make_read_tool(ws)], "What port?")
print(lab.show([reply]))

KeyError: 'raed'

Somebody was in a position to turn raed into read. Who, and did they ever find out? The cell prints what your terminal was told, and then every request the model received.

import harness, lab

ws = lab.Workspace({"config.py": "PORT = 9090\n"})
model = lab.ScriptedModel([
    lab.reply(lab.call("raed", {"path": "config.py"})),
    lab.say("The port is 9090."),
])

messages = []
try:
    harness.run_agent(model, "You are helpful.", messages,
                      [harness.make_read_tool(ws)], "What port?")
except KeyError as exc:
    print("your terminal got: KeyError:", exc)

print("the model got, in all its requests:")
for number, request in enumerate(model.calls, 1):
    print(f"  request {number}:", lab.show(request.messages))
print("requests that mention the typo:",
      sum("raed" in lab.show(r.messages) for r in model.calls))

your terminal got: KeyError: 'raed'
the model got, in all its requests:
  request 1: user -> "What port?"
requests that mention the typo: 0

Which design comes back with an answer on all three paths? Think about the third one in particular: the author of read wrote an except FileNotFoundError, because that is the failure they had in mind.

import lab

ws = lab.Workspace({"config.py": "PORT = 9090\n",
                    "logo.png": b"\x89PNG\xff\xfe"})

# Design A: every tool reports its own failures with a flag.
def read_with_flag(arguments):
    try:
        return {"ok": True, "text": ws.read_text(arguments["path"])}
    except FileNotFoundError:          # the failure its author foresaw
        return {"ok": False, "text": "File not found."}

# Design B: tools just raise. One boundary, outside every tool.
def read_that_raises(arguments):
    return ws.read_text(arguments["path"])

def boundary(execute, arguments):
    try:
        return {"ok": True, "text": execute(arguments)}
    except Exception as exc:
        return {"ok": False, "text": type(exc).__name__}

for path in ["config.py", "confg.py", "logo.png"]:
    print(path)
    try:
        print("  A:", read_with_flag({"path": path})["ok"])
    except Exception as exc:
        print("  A: the run died,", type(exc).__name__)
    print("  B:", boundary(read_that_raises, {"path": path})["ok"])

config.py
  A: True
  B: True
confg.py
  A: False
  B: False
logo.png
  A: the run died, UnicodeDecodeError
  B: False

Pick the text you would write, then read all three runs. Which one lets the job finish?

import harness, lab

FILES = {"config.py": "PORT = 9090\n", "main.py": "import config\n"}

TRACEBACK = "\n".join(
    ["Traceback (most recent call last):"]
    + [line for frame in range(19)
       for line in (f'  File "harness.py", line {62 + frame}, in run_tool',
                    '    content = tool["execute"](call["arguments"])')]
    + ["FileNotFoundError: confg.py"])

def read_tool(ws, message):
    """The read tool, with the error text under test. `message` takes the path."""
    def execute(arguments):
        path = harness.str_arg(arguments, "path")
        if not ws.exists(path):
            raise FileNotFoundError(message(path))
        return ws.read_text(path)
    return {**harness.make_read_tool(ws), "execute": execute}

def debugger(request):
    """Our script. It asks for confg.py. After a failed call it looks in the error
    text for a file name it has not tried yet and tries that; if there is none it
    gives up. It has no other source of file names."""
    result = request.last_result
    if result is None:
        return lab.reply(lab.call("read", {"path": "confg.py"}))
    if not result["is_error"]:
        return lab.say("Found it: " + result["content"].strip())
    asked = [block["arguments"]["path"] for message in request.messages
             if message["role"] == "assistant"
             for block in message["content"] if block["type"] == "toolCall"]
    fresh = [word.strip('",.()') for word in result["content"].split()
             if word.strip('",.()').endswith(".py")
             and word.strip('",.()') not in asked]
    if fresh:
        return lab.reply(lab.call("read", {"path": fresh[0]}))
    return lab.say("I cannot continue: the error does not tell me what to try.")

TEXTS = [
    ("Error", lambda path: "Error"),
    (f"a traceback ({len(TRACEBACK.splitlines())} lines)",
     lambda path: TRACEBACK.replace("confg.py", path)),
    ("File not found: confg.py. Files here: config.py, main.py",
     lambda path: f"File not found: {path}. "
                  f"Files here: {', '.join(lab.Workspace(FILES).listdir('.'))}"),
]

for label, message in TEXTS:
    ws = lab.Workspace(FILES)
    model = lab.ScriptedModel([lab.forever(debugger)])
    messages = []
    reply = harness.run_agent(model, "You are a careful debugger.", messages,
                              [read_tool(ws, message)], "What port does the app use?")
    asked = [block["arguments"]["path"] for m in messages if m["role"] == "assistant"
             for block in m["content"] if block["type"] == "toolCall"]
    print(f'error text: {label}')
    print(f'  paths it asked for: {", ".join(asked)}')
    print(f'  model calls: {len(model.calls)}')
    print(f'  it ended with: {harness.text_of(reply)}')

error text: Error
  paths it asked for: confg.py
  model calls: 2
  it ended with: I cannot continue: the error does not tell me what to try.
error text: a traceback (40 lines)
  paths it asked for: confg.py, harness.py
  model calls: 3
  it ended with: I cannot continue: the error does not tell me what to try.
error text: File not found: confg.py. Files here: config.py, main.py
  paths it asked for: confg.py, config.py
  model calls: 3
  it ended with: Found it: PORT = 9090

The boundary from the last section is in place, and str_arg is that one line. The model sends bash({}) and read({"path": 7}). What comes back?

import harness, lab

# The str_arg you were given, word for word. It checks nothing.
harness.str_arg = lambda arguments, name: arguments[name]

ws = lab.Workspace({"config.py": "PORT = 9090\n"})
tools = [harness.make_read_tool(ws), harness.make_bash_tool(lab.Shell(ws))]

def row(label, value):
    print(f'  {label:<31}{value}')

texts = {}
for call, wanted in [(lab.call("bash", {}, id="c1"), "command"),
                     (lab.call("read", {"path": 7}, id="c2"), "path")]:
    result = harness.run_tool(tools, call)
    text = texts[call["name"]] = result["content"]
    print(f'{call["name"]}({call["arguments"]})')
    row("is_error", result["is_error"])
    row("length of the text", f"{len(text)} characters")
    row(f'does it name "{wanted}"?', wanted in text)
    row("does it say what to send?", "string" in text)

print()
print("the whole of what bash's failure told the model:", repr(texts["bash"]))

bash({})
  is_error                       True
  length of the text             9 characters
  does it name "command"?        True
  does it say what to send?      False
read({'path': 7})
  is_error                       True
  length of the text             43 characters
  does it name "path"?           False
  does it say what to send?      False

the whole of what bash's failure told the model: "'command'"

The command exited with code 1. Your bash tool has to turn that into a result. Is it flagged as an error? The cell runs the same model against both answers.

import harness, lab

def bash_tool(shell, *, exit_code_is_an_error):
    """The given bash tool, in two designs, differing only in the three lines below."""
    def execute(arguments):
        output, code = shell.run(harness.str_arg(arguments, "command"))
        if code != 0:
            if exit_code_is_an_error:
                raise RuntimeError(f"Command exited with code {code}")
            output += f"\nCommand exited with code {code}"
        return output
    return {**harness.make_bash_tool(shell), "execute": execute}

def developer(request):
    """Our script. It runs the tests once. If the result comes back flagged as an
    error it stops, because a broken tool is not something it can work around.
    Otherwise it reads the output and reports the failing line."""
    result = request.last_result
    if result is None:
        return lab.reply(lab.call("bash", {"command": "pytest"}))
    if result["is_error"]:
        return lab.say("The bash tool failed, so I never saw the test output. Stopping.")
    failed = [line for line in result["content"].splitlines() if line.startswith("E ")]
    return lab.say("One test failed: " + failed[-1].split("assert")[-1].strip())

for flags in [True, False]:
    ws = lab.Workspace({"calc.py": "def total(xs):\n    return sum(xs) + 1\n"})
    tool = bash_tool(lab.Shell(ws), exit_code_is_an_error=flags)
    model = lab.ScriptedModel([lab.forever(developer)])
    messages = []
    reply = harness.run_agent(model, "You are a careful developer.", messages,
                              [tool], "Run the tests and tell me what is wrong.")
    result = messages[2]
    print(f'exit_code_is_an_error={flags}')
    print(f'  is_error        {result["is_error"]}')
    print(f'  lines of output {len(result["content"].splitlines())}')
    print(f'  last line       {result["content"].splitlines()[-1]!r}')
    print(f'  it ended with   {harness.text_of(reply)}')

exit_code_is_an_error=True
  is_error        True
  lines of output 1
  last line       'Command exited with code 1'
  it ended with   The bash tool failed, so I never saw the test output. Stopping.
exit_code_is_an_error=False
  is_error        False
  lines of output 3002
  last line       'Command exited with code 1'
  it ended with   One test failed: 5 == 6

Whatever comes out of tool["execute"](...), your boundary turns it into a result and the run carries on. Should that except catch everything?

import lab

# Four things that can come out of tool["execute"](...). Which does `except
# Exception` catch, and which would only a bare `except:` catch?
things = [
    FileNotFoundError("File not found: confg.py. Files here: config.py"),
    ValueError("path must be a string"),
    KeyError("port"),
    lab.PowerCut("the power went out inside a tool"),
]
for thing in things:
    print(f'{type(thing).__name__:<18} caught by "except Exception": '
          f'{isinstance(thing, Exception)}')

FileNotFoundError  caught by "except Exception": True
ValueError         caught by "except Exception": True
KeyError           caught by "except Exception": True
PowerCut           caught by "except Exception": False