diff --git a/Gemfile b/Gemfile index 7ad261d..7f8cbe5 100644 --- a/Gemfile +++ b/Gemfile @@ -77,3 +77,4 @@ end gem "csv" gem "ruby_llm" +gem "faraday" diff --git a/Gemfile.lock b/Gemfile.lock index 78efdf1..dd8a0a0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -462,6 +462,7 @@ DEPENDENCIES dotenv-rails (~> 3.2) factory_bot_rails faker + faraday image_processing (~> 1.2) importmap-rails jbuilder diff --git a/README.md b/README.md index 82ccb63..c1a2de0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Status: learning project

-> **Status:** Lessons 1 to 3 of 14 are done: real and planted data, the first LLM call, and four read tools the model chooses between. Lesson 4 (the agent loop, written by hand) is next. This is a learning project, built step by step, not a finished product. The plan below describes where it is going, not what works today. +> **Status:** Lessons 1 to 4 of 14 are done: real and planted data, the first LLM call, four read tools the model chooses between, and the agent loop written by hand. Lesson 5 (an MCP server over the same tools) is next. This is a learning project, built step by step, not a finished product. The plan below describes where it is going, not what works today. ## The problem @@ -120,7 +120,7 @@ For a quick try, `LIMIT=200 bin/rails data:import_olist` loads only 200 customer | Observability | OpenTelemetry, Langfuse | | Evals | eval-ruby, rspec-agents | -Gems are added in the lesson that needs them, not all at once. All LLM calls will go through one small `LlmClient` class, so changing provider is a config change. +Gems are added in the lesson that needs them, not all at once. The hand-written loop in Lesson 4 talks to Groq through one small `LlmClient`. From Lesson 5 on the app uses RubyLLM, so changing provider is a config change. ## Roadmap @@ -192,5 +192,6 @@ This is a learning project about building a reliable AI agent, not a complete su - Lesson progress, decisions and what broke: [docs/lessons/01-data-model-and-import.md](docs/lessons/01-data-model-and-import.md) - Lesson 2, the first LLM call: [docs/lessons/02-first-llm-call.md](docs/lessons/02-first-llm-call.md) - Lesson 3, tool calling with read tools: [docs/lessons/03-read-tools.md](docs/lessons/03-read-tools.md) +- Lesson 4, the agent loop written by hand: [docs/lessons/04-agent-loop.md](docs/lessons/04-agent-loop.md) - Index of all documentation: [docs/README.md](docs/README.md) - Where the code lives: this GitHub repo is the source of truth, and pull requests and issues are here. A read-only mirror is kept on [GitLab](https://gitlab.com/mjesar/support_engineer). diff --git a/app/agents/README.md b/app/agents/README.md new file mode 100644 index 0000000..6ea8ae9 --- /dev/null +++ b/app/agents/README.md @@ -0,0 +1,12 @@ +# app/agents + +The agent code. There are two ways to run the same agent here, on purpose. + +| File | Role | +| --- | --- | +| `support_agent.rb` | **The agent the app uses.** A RubyLLM agent: model, tool allowlist and prompt. `chat.ask` runs the loop. | +| `agent_loop.rb`, `llm_client.rb`, `agent_tools.rb` | **The Lesson 4 hand-written version,** kept for learning. The same loop in plain Ruby with a step limit and a trace. Nothing else depends on it. | +| `triage.rb`, `next_step.rb` | Routing: classify a message, then pick the next step. | + +Both versions use the same tool classes in `app/tools/`. See +[Lesson 4](../../docs/lessons/04-agent-loop.md) for how they compare. diff --git a/app/agents/agent_loop.rb b/app/agents/agent_loop.rb new file mode 100644 index 0000000..d812ee2 --- /dev/null +++ b/app/agents/agent_loop.rb @@ -0,0 +1,47 @@ +# The hand-written agent loop: ask the model, run the tools it requests, give it the results, +# and repeat until it answers in plain text. It owns the messages (the context); LlmClient +# and AgentTools stay stateless. +class AgentLoop + # stop_reason is :answered or :step_limit, so the caller can tell a real answer from a cut-off. + # trace lists every model request and tool call in order, to see afterwards what the agent did. + Result = Struct.new(:answer, :stop_reason, :trace) + + def initialize(client:, tools:, max_steps: 5) + @client = client + @tools = tools + @max_steps = max_steps + end + + def run(question) + messages = [ { role: "user", content: question } ] + trace = [] + + # The model decides when to stop, so a step limit keeps a confused one from looping forever. + @max_steps.times do |step| + trace << { event: :model_request, step: step + 1, message_count: messages.size } + reply = @client.chat(messages: messages, tools: @tools.definitions) + messages << reply + return Result.new(reply["content"], :answered, trace) unless reply["tool_calls"] + + reply["tool_calls"].each { |tool_call| messages << tool_message(tool_call, trace) } + end + + Result.new(nil, :step_limit, trace) + end + + private + + def tool_message(tool_call, trace) + function = tool_call["function"] + result = run_tool(function) + trace << { event: :tool_call, name: function["name"], arguments: function["arguments"], result: result } + { role: "tool", tool_call_id: tool_call["id"], content: result.to_json } + end + + # Bad arguments come back as an error the model can read and fix, like an unknown tool name. + def run_tool(function) + @tools.run(function["name"], JSON.parse(function["arguments"])) + rescue JSON::ParserError + { error: "Arguments for #{function['name']} are not valid JSON" } + end +end diff --git a/app/agents/agent_tools.rb b/app/agents/agent_tools.rb new file mode 100644 index 0000000..0e33e39 --- /dev/null +++ b/app/agents/agent_tools.rb @@ -0,0 +1,28 @@ +# The tools the loop may use. The wrapper classes stay the only place that describes a tool; +# this class turns them into what Groq expects and runs them by name. +class AgentTools + def initialize(tool_classes) + @tools = tool_classes.map(&:new).index_by(&:name) + end + + def definitions + @tools.values.map { |tool| definition(tool) } + end + + # Unknown names come back as an error hash so the model can see its mistake and recover. + def run(name, arguments) + tool = @tools[name] + return { error: "Unknown tool #{name}" } unless tool + + tool.execute(**arguments.symbolize_keys) + end + + private + + # "strict" is dropped: RubyLLM sets it at a different level in its own requests. + def definition(tool) + { type: "function", + function: { name: tool.name, description: tool.description, + parameters: tool.parameters_schema.except("strict") } } + end +end diff --git a/app/agents/llm_client.rb b/app/agents/llm_client.rb new file mode 100644 index 0000000..6a8aa4b --- /dev/null +++ b/app/agents/llm_client.rb @@ -0,0 +1,26 @@ +# One request to Groq: sends the messages and tool definitions, returns the assistant message. +# It does not run tools or keep the conversation; that is AgentLoop's job. +class LlmClient + BASE_URL = "https://api.groq.com/openai/v1".freeze + MODEL = "openai/gpt-oss-120b".freeze + + def chat(messages:, tools:) + response = connection.post("chat/completions", request_body(messages, tools)) + response.body["choices"][0]["message"] + end + + private + + # include_reasoning: false because Groq rejects the reasoning field when it is sent back. + def request_body(messages, tools) + { model: MODEL, messages: messages, tools: tools, include_reasoning: false } + end + + def connection + @connection ||= Faraday.new(BASE_URL, headers: { "Authorization" => "Bearer #{ENV.fetch('GROQ_API_KEY')}" }) do |faraday| + faraday.request :json + faraday.response :json + faraday.response :raise_error + end + end +end diff --git a/assets/readme/hand-written-loop.svg b/assets/readme/hand-written-loop.svg new file mode 100644 index 0000000..9f4ba02 --- /dev/null +++ b/assets/readme/hand-written-loop.svg @@ -0,0 +1,57 @@ + + The hand-written agent loop + AgentLoop keeps a messages list and repeats three steps. It asks the model through LlmClient, then checks whether the reply has tool calls. If yes, AgentTools runs each requested tool, the results are added to messages, and the model is asked again. If no, the reply content is the final answer and the loop stops. Planned safety rails: a step limit, unknown tool names and invalid JSON arguments. + + + + + + + The hand-written agent loop + The model decides how many rounds happen. The loop only repeats until it stops asking for tools. + + + + 1. Ask the model + LlmClient.chat + sends all messages + tools + + + + 2. Reply has + tool_calls? + + + + 3. Run the tools + AgentTools.run + one tool message per call + + + + Stop: the answer + return the reply content + + + + + yes + + no + + add the results to messages, then ask again + + + + messages (the context) + user question + assistant reply (with tool_calls) + tool result, linked by tool_call_id + + + + Safety rails (the model cannot be trusted to stop) + Step limit, with a stop reason + Unknown tool name: error back to the model + Arguments that are not valid JSON: error back too + diff --git a/docs/FAQ.md b/docs/FAQ.md index e3290b0..2b3fd28 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -101,3 +101,111 @@ or a local `stdio` process. Hosted servers usually need OAuth or a bearer token. This project does not use it yet (checked 2026-10-10): the course plan has the MCP server side in Lesson 5 and no lesson where the agent calls an outside server. The `ai_shop_assistant` project does this against Shopify's Catalog API. + +## What is `AgentTools` for in the hand-written agent loop? + +It is the bridge between the model and the Ruby tool classes. The model only reads and writes text, so +two translations are needed, and `AgentTools` does both: + +1. **Code to model.** `definitions` turns the tool wrapper classes into the JSON `tools` list that + Groq expects (name, description, argument schema). It is sent with every request. +2. **Model to code.** `run(name, arguments)` takes the tool name the model asked for (`"get_order"`) and + its parsed arguments, finds the matching tool and executes it. + +It is also the allowlist. The model can only run the tools passed to `AgentTools.new`; an invented name +such as `delete_order` returns an error hash and nothing runs. + +``` +AgentLoop keeps the messages, repeats until the model answers + ├─ LlmClient talks to Groq + └─ AgentTools lists the tools and runs them by name + └─ GetOrderTool → GetOrder → database +``` + +The wrapper classes (`RubyLLM::Tool`) stay the only place that describes a tool, so the hand-written +loop and `SupportAgent` send the model identical tool descriptions. Only the declarations are reused; +the loop itself is hand-written. `AgentLoop` parses the raw `arguments` JSON string before calling +`run`, so `AgentTools` only ever sees a hash. + +## What is an agent loop, and how does it work? + +An agent loop is a `loop` around the model call. The model can only read and write text, so it cannot +look anything up itself. It asks for a tool, the code runs it, and the result goes back to the model. +This repeats until the model answers in plain text. + +The model has no memory between calls, so the loop keeps the `messages` array (the context) and sends +all of it every time. Each round adds to it: + +1. The user question is added. +2. The model replies with `tool_calls` (a request, no `content`) or with `content` (the answer). +3. The model's reply is added to `messages`, so the history stays complete. +4. For each tool call, the code runs the tool and adds a `tool` message. Its `tool_call_id` links the + result to the request it answers, and its `content` must be a string (`result.to_json`). +5. The model is called again with the longer `messages`. + +``` +ask model → tool_calls? → yes → run tools, add results to messages → ask model again + → no → the content is the answer, stop +``` + +The model decides how many rounds happen, which is what makes it an agent and not a fixed pipeline. A +normal LLM call is one round trip. An agent loop is many, with tool results in between. + +The stop condition is "the reply has no `tool_calls`". Because the model controls it, the loop needs +safety rails: a step limit (with a stop reason), unknown tool names, and arguments that are not valid +JSON. Without a step limit, a model that keeps asking for tools loops forever. + +Frameworks such as RubyLLM run this same loop inside `chat.ask`. Writing it by hand once (Lesson 4) +shows what they hide. + +## Are safety rails and guardrails the same thing? + +They overlap and people use the words loosely, but in this project they mean two layers. + +**Safety rails** are technical limits that keep the agent loop from breaking. They do not judge what +the agent says. Examples: a step limit so the loop cannot run forever, an unknown tool name returning an +error instead of running something, and invalid JSON arguments being caught instead of crashing the +run. They matter even if the model behaves well, because it can still make mechanical mistakes. + +**Guardrails** is the broader industry term for rules about content and behavior: + +- Input checks: block prompt injection, detect personal data, refuse off-topic requests. +- Output checks: do not leak another customer's data, do not promise a refund the policy does not allow. +- Action limits: a human approves risky actions (the approval screen for write tools, Lesson 9). +- Permissions: the allowlist of tools, and read tools kept apart from write tools. + +| | Safety rails | Guardrails | +| --- | --- | --- | +| Question | Can the loop run without breaking? | Should the agent say or do this? | +| Example | Stop after 5 steps | A refund over a limit needs approval | +| Where | Inside `AgentLoop` | Around it, and in tools, prompts and approval screens | + +Guardrails include some safety rails: the `AgentTools` allowlist is both a loop-robustness feature and a +guardrail on what the model may do. + +## Why write the agent loop by hand if RubyLLM already does it? + +"Hand-written" means the loop is plain Ruby in this repo (`AgentLoop`), not hidden inside a library. +The result is the same. The difference is whether you can see it. + +- **With RubyLLM:** `chat.ask(question)` runs the whole loop in one call. It sends the messages, sees + `tool_calls`, runs the tool, adds the result and asks again until the model answers. The loop is + there, but you never see it. +- **By hand (Lesson 4):** `AgentLoop` is the same loop in about 25 lines. We build the `messages` + array, call Groq through Faraday, parse the arguments and add each tool result ourselves. + +Reasons to write it once: + +- **Understanding.** A hidden loop is hard to debug. After writing it, you know what a framework does + when it misbehaves. +- **Interviews.** "How does an agent work?" is easy to answer when you have built the loop: what + `tool_calls` looks like, why the reply goes back into the history, why a step limit is needed. +- **Failure modes.** Writing the safety rails (step limit, invalid JSON, unknown tool) shows why a + framework needs them too. + +It is not the better way for real work. From Lesson 5 on the project uses RubyLLM, which also handles +streaming, retries, structured output and several providers, none of which are worth rebuilding. + +The lesson ends with a check: the same question goes through `AgentLoop` and through `SupportAgent` +(RubyLLM), and both should make the same tool calls. If they do, the framework is doing what the +hand-written loop does. diff --git a/docs/PROJECT_PLAN.md b/docs/PROJECT_PLAN.md index 63d9077..0e686d0 100644 --- a/docs/PROJECT_PLAN.md +++ b/docs/PROJECT_PLAN.md @@ -266,11 +266,11 @@ The project choice is locked. New project plans from other tools are not conside - [x] Lesson 3: tool calling with read tools, tagged `lesson-03`. Details: [lessons/03-read-tools.md](lessons/03-read-tools.md) - [ ] Lesson 4: the agent loop, written by hand with no framework (branch `lesson-04-agent-loop`) - [x] Read Groq's tool-calling format in its docs, then send one raw request in the console: `messages`, `tools`, the `tool_calls` in the reply, and the `tool` message that answers it - - [ ] `LlmClient`: one small class that posts the messages and tool definitions to Groq and returns the assistant message - - [ ] Tool definitions for the loop, and a name to service map that runs the four existing services - - [ ] `AgentLoop`: keeps the messages (the context), calls the model, runs any tool calls, appends the results, repeats until the model answers - - [ ] Safety rails: a step limit with a stop reason, an unknown tool name, arguments that are not valid JSON - - [ ] A trace of the run: each model request, each tool call and its result, and the stop reason - - [ ] Console proof: the same question through `AgentLoop` and through `SupportAgent` makes the same tool calls - - [ ] Specs: one recorded run (order 4242), the step limit and the unknown tool with a stubbed client + - [x] `LlmClient`: one small class that posts the messages and tool definitions to Groq and returns the assistant message + - [x] Tool definitions for the loop, and a name to service map that runs the four existing services + - [x] `AgentLoop`: keeps the messages (the context), calls the model, runs any tool calls, appends the results, repeats until the model answers + - [x] Safety rails: a step limit with a stop reason, an unknown tool name, arguments that are not valid JSON + - [x] A trace of the run: each model request, each tool call and its result, and the stop reason + - [x] Console proof: the same question through `AgentLoop` and through `SupportAgent` makes the same tool calls + - [x] Specs: one recorded run (order 4242), the step limit and the unknown tool with a stubbed client - [ ] Lesson page with a loop diagram, a FAQ entry on the agent loop, a gem-gap line in `~/.claude/IDEAS.md` (RubyLLM already covers this), tag `lesson-04` diff --git a/docs/README.md b/docs/README.md index e0da28a..58ef5a8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ lesson pages in order. | [lessons/01-data-model-and-import.md](lessons/01-data-model-and-import.md) | Lesson 1: progress checklist, decisions, what broke and measured results. | | [lessons/02-first-llm-call.md](lessons/02-first-llm-call.md) | Lesson 2: Groq through RubyLLM 2.x, the agent and its prompt, testing with recorded calls. | | [lessons/03-read-tools.md](lessons/03-read-tools.md) | Lesson 3: four read tools, how data reaches the agent (diagram), design decisions, what broke. | +| [lessons/04-agent-loop.md](lessons/04-agent-loop.md) | Lesson 4: the agent loop written by hand (diagram), safety rails, a trace, and a comparison with RubyLLM. | ## Conventions diff --git a/docs/lessons/03-read-tools.md b/docs/lessons/03-read-tools.md index 553a621..301f4b5 100644 --- a/docs/lessons/03-read-tools.md +++ b/docs/lessons/03-read-tools.md @@ -72,7 +72,7 @@ model answers from the result "Your order #1 has been shipped . ``` `chat.ask` runs this whole loop itself and stops when the model answers instead of asking for a tool. -Lesson 4 writes the same loop by hand, one step at a time, using `step`, `generate` and `run_tools`. +[Lesson 4](04-agent-loop.md) writes the same loop by hand, in plain Ruby, and checks that it makes the same tool calls. What the model reads to choose a tool is the tool's `description` and argument schema, nothing else. In the console, "Where is the package for my order 1?" led to `get_shipment`, and "I am customer 68585. diff --git a/docs/lessons/04-agent-loop.md b/docs/lessons/04-agent-loop.md new file mode 100644 index 0000000..003b322 --- /dev/null +++ b/docs/lessons/04-agent-loop.md @@ -0,0 +1,140 @@ +# Lesson 4: the agent loop, written by hand + +A framework hides the loop that makes an agent an agent. This lesson writes it in about 25 lines of plain +Ruby: ask the model, run the tools it requests, give it the results, repeat until it answers. Then it checks +that the hand-written loop makes the same tool calls as `SupportAgent` (RubyLLM). + +Last updated: 2026-10-12 + +## Status + +- [x] Groq's tool-calling format read in its docs, then one raw request sent from the console +- [x] `LlmClient`: one request to Groq, returns the assistant message +- [x] `AgentTools`: tool definitions for the request, and a run-by-name allowlist over the four tool classes +- [x] `AgentLoop`: keeps the messages, repeats until the model answers +- [x] Safety rails: step limit with a stop reason, unknown tool name, arguments that are not valid JSON +- [x] A trace of each run +- [x] Console proof: `AgentLoop` and `SupportAgent` make the same tool call for the same question +- [x] Specs: six with a fake client, one recorded run +- [ ] Pull request merged and tag `lesson-04` + +## How the loop works + +

+ Diagram of the hand-written agent loop: AgentLoop asks the model through LlmClient, checks whether the reply has tool calls, runs each requested tool through AgentTools, adds the results to the messages and asks again. When the reply has no tool calls, its content is the final answer. Safety rails: a step limit, unknown tool names and invalid JSON arguments. +

+ +
+Text version + +`AgentLoop` keeps one list, `messages`, and repeats three steps: + +1. **Ask the model.** `LlmClient.chat` sends all the messages and the tool definitions to Groq. The model + has no memory between calls, so the whole list goes every time. +2. **Check the reply.** If it has `tool_calls`, the model is asking for help. If it does not, its `content` + is the final answer and the loop stops. +3. **Run the tools.** `AgentTools.run` finds each requested tool by name and runs it. The model's reply and + one `tool` message per call (linked by `tool_call_id`) are added to `messages`, and the loop goes back to + step 1. + +Safety rails around the loop: a step limit (`max_steps`, default 5) ends a run that never answers, an +unknown tool name returns an error to the model instead of running anything, and arguments that are not +valid JSON also return an error to the model instead of crashing. + +
+ +## What was built + +``` +app/agents/llm_client.rb one request to Groq (Faraday), returns the assistant message +app/agents/agent_tools.rb tool definitions for the request, and run(name, arguments) by name +app/agents/agent_loop.rb the loop, the step limit, the error handling and the trace +app/agents/README.md which agent the app uses and which one is the lesson +spec/agents/agent_loop_spec.rb six specs with a fake client, one recorded run +``` + +`AgentLoop.new(client:, tools:, max_steps: 5).run(question)` returns a `Result` with the `answer`, a +`stop_reason` (`:answered` or `:step_limit`) and the `trace`. + +## Hand-written versus RubyLLM + +Both versions run the same four tool classes with the same Groq model. In the console, "Where is my order +4242?" gave the same tool call on both sides: + +``` +SupportAgent get_shipment {"order_id" => 4242} +AgentLoop get_shipment {"order_id":4242} +``` + +The only visible difference is that RubyLLM hands the arguments over already parsed, and the hand-written +loop parses the JSON string itself. With only `GetOrderTool` attached, the model chose `get_order` instead. +The tool descriptions are what it reads, so the choice depends on which tools it is offered. + +| | `SupportAgent` (RubyLLM) | `AgentLoop` (by hand) | +| --- | --- | --- | +| Role | the agent the app uses | the lesson, nothing depends on it | +| Who repeats the loop | `chat.ask` | our `@max_steps.times` | +| Provider and HTTP | RubyLLM, configured in an initializer | one hard-coded Groq client | +| System prompt | `app/prompts/support_agent` | none yet | +| Streaming, retries, other providers | yes | no | + +## Design decisions + +| Decision | Choice | Why | +| --- | --- | --- | +| Who owns the messages | `AgentLoop` (a local array in `run`) | `LlmClient` and `AgentTools` stay stateless, so each is easy to test and replace. | +| Client passed in | `AgentLoop.new(client:, tools:)` | Specs hand it a fake client, so no spec needs Groq except the recorded one. | +| Tool descriptions | built from the existing `RubyLLM::Tool` wrappers | One place describes a tool, so both versions send the model the same words. | +| Errors in tools | a Hash the model can read, never a raise | The model can see its mistake and retry, the same rule the tools already follow. | +| Return value | `Result` struct with `answer`, `stop_reason`, `trace` | A cut-off run must not look like an answer. | +| Trace | one entry per model request and per tool call, raw arguments kept | Bad arguments show up exactly as the model sent them. | + +## What broke + +- **Reasoning text sent back.** Groq rejects the `reasoning` field of a message when it is sent back in + the next request, so `LlmClient` sends `include_reasoning: false`. `SupportAgent` needs the same option + for the same reason. +- **A reply with no `content`.** When the model asks for a tool, Groq's message has `tool_calls` and no + `content` key at all, so the loop decides on `tool_calls`, not on `content`. +- **The raw arguments are a string.** Groq sends `"arguments":"{\"order_id\":4242}"` as a JSON string, so + the loop must parse it, and the parse can fail. + +## Proof + +Six specs use a fake client that plays back canned replies, so they run without Groq: a plain answer, a +tool call whose result reaches the model, the step limit, an unknown tool, invalid JSON arguments (the tool +does not run), and the trace. + +One recorded run (`spec/cassettes/agent_loop/looks_up_an_order.yml`) uses the real model with all four +tools, order 4242 and fixed dates. It holds three requests: + +``` +request 1: question -> model: get_shipment(order_id: 4242) +request 2: ... + get_shipment result -> model: get_order(order_id: 4242) +request 3: ... + get_order result -> final answer +``` + +The order had no shipment, so the first tool returned `{"error":"Order 4242 has no shipment"}`. The model +read that, asked for `get_order` on its own, and then answered. Nobody wrote that second round into the +code; the loop only repeated until the model stopped asking. The spec checks the first tool call's +arguments and the `:answered` stop reason, not the number of calls, because that can change between +recordings. + +## Known gaps + +- `AgentLoop` sends no system prompt, so it does not follow `SupportAgent`'s instructions. +- Arguments that are valid JSON but have the wrong keys raise an `ArgumentError` inside the tool call and + stop the run. Only invalid JSON and unknown tool names are handled. +- The loop is tied to Groq's Chat Completions format through `LlmClient`. + +## Scope + +This shows that the loop works and matches the framework on one question. It does not measure how often the +model picks the right tool or whether its answers stay grounded. That is the job of the evals in Lesson 10. + +## Learn more + +- [FAQ](../FAQ.md): what an agent loop is, what `AgentTools` is for, safety rails versus guardrails, and + why the loop is hand-written +- [Lesson 3](03-read-tools.md): the four tools the loop runs +- [app/agents/README.md](../../app/agents/README.md): which agent the app uses diff --git a/spec/agents/agent_loop_spec.rb b/spec/agents/agent_loop_spec.rb new file mode 100644 index 0000000..6b7eb89 --- /dev/null +++ b/spec/agents/agent_loop_spec.rb @@ -0,0 +1,111 @@ +require "rails_helper" + +RSpec.describe AgentLoop do + # Plays back canned replies in order, so the loop is tested without calling Groq. + # It keeps a copy of the messages it was sent each time, to check what the model would have seen. + let(:fake_client_class) do + Class.new do + attr_reader :calls + + def initialize(replies) + @replies = replies + @calls = [] + end + + def chat(messages:, tools:) + @calls << messages.dup + @replies.shift + end + end + end + + let(:tools) { AgentTools.new([ GetOrderTool ]) } + + def tool_call_reply(name, arguments) + { "role" => "assistant", + "tool_calls" => [ { "id" => "call_1", "type" => "function", + "function" => { "name" => name, "arguments" => arguments } } ] } + end + + def answer_reply(text) + { "role" => "assistant", "content" => text } + end + + it "returns the answer when the model does not ask for a tool" do + client = fake_client_class.new([ answer_reply("Hello!") ]) + + result = described_class.new(client: client, tools: tools).run("Say hi") + + expect(result.answer).to eq("Hello!") + expect(result.stop_reason).to eq(:answered) + end + + it "runs the requested tool and gives the result back to the model" do + create(:order, id: 4242, status: "shipped") + client = fake_client_class.new([ tool_call_reply("get_order", '{"order_id":4242}'), answer_reply("It shipped.") ]) + + result = described_class.new(client: client, tools: tools).run("Where is my order 4242?") + + tool_message = client.calls.last.last + expect(tool_message).to include(role: "tool", tool_call_id: "call_1") + expect(JSON.parse(tool_message[:content])).to include("id" => 4242, "status" => "shipped") + expect(result.answer).to eq("It shipped.") + end + + it "stops with :step_limit when the model keeps asking for tools" do + replies = Array.new(3) { tool_call_reply("get_order", '{"order_id":1}') } + client = fake_client_class.new(replies) + + result = described_class.new(client: client, tools: tools, max_steps: 3).run("Where is my order 1?") + + expect(result.stop_reason).to eq(:step_limit) + expect(result.answer).to be_nil + expect(client.calls.size).to eq(3) + end + + it "sends an unknown tool name back as an error and lets the model recover" do + client = fake_client_class.new([ tool_call_reply("delete_order", '{"order_id":1}'), answer_reply("I cannot do that.") ]) + + result = described_class.new(client: client, tools: tools).run("Delete order 1") + + expect(client.calls.last.last[:content]).to include("Unknown tool delete_order") + expect(result.stop_reason).to eq(:answered) + end + + it "sends arguments that are not valid JSON back as an error without running the tool" do + client = fake_client_class.new([ tool_call_reply("get_order", "not json"), answer_reply("Sorry, let me retry.") ]) + allow(GetOrder).to receive(:call) + + result = described_class.new(client: client, tools: tools).run("Where is my order 1?") + + expect(GetOrder).not_to have_received(:call) + expect(client.calls.last.last[:content]).to include("not valid JSON") + expect(result.stop_reason).to eq(:answered) + end + + it "records every model request and tool call in the trace" do + create(:order, id: 4242) + client = fake_client_class.new([ tool_call_reply("get_order", '{"order_id":4242}'), answer_reply("Done.") ]) + + result = described_class.new(client: client, tools: tools).run("Where is my order 4242?") + + expect(result.trace.map { |entry| entry[:event] }).to eq(%i[model_request tool_call model_request]) + expect(result.trace[1]).to include(name: "get_order", arguments: '{"order_id":4242}') + expect(result.trace.last[:message_count]).to eq(3) + end + + # A fixed id and fixed dates keep the request body identical on every run, so the recording + # replays on any day (see spec/agents/support_agent_spec.rb). + it "answers a real question through the real model", vcr: { cassette_name: "agent_loop/looks_up_an_order" } do + create(:order, id: 4242, status: "shipped", purchased_at: Time.utc(2026, 9, 20), + shipped_at: Time.utc(2026, 9, 24), delivered_at: nil, + estimated_delivery_at: Time.utc(2026, 10, 1)) + all_tools = AgentTools.new([ GetOrderTool, GetShipmentTool, GetCustomerTool, SearchOrdersTool ]) + + result = described_class.new(client: LlmClient.new, tools: all_tools).run("Where is my order 4242?") + + tool_calls = result.trace.select { |entry| entry[:event] == :tool_call } + expect(JSON.parse(tool_calls.first[:arguments])).to eq("order_id" => 4242) + expect(result.stop_reason).to eq(:answered) + end +end diff --git a/spec/cassettes/agent_loop/looks_up_an_order.yml b/spec/cassettes/agent_loop/looks_up_an_order.yml new file mode 100644 index 0000000..8ad4b52 --- /dev/null +++ b/spec/cassettes/agent_loop/looks_up_an_order.yml @@ -0,0 +1,274 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.groq.com/openai/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"openai/gpt-oss-120b","messages":[{"role":"user","content":"Where + is my order 4242?"}],"tools":[{"type":"function","function":{"name":"get_order","description":"Looks + up one order by its id. Returns its status, purchase, shipping and delivery + dates, and the items with name, category, price and whether they are final + sale.","parameters":{"type":"object","properties":{"order_id":{"type":"integer","description":"The + order id the customer gives, for example 1042"}},"required":["order_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"get_shipment","description":"Looks + up where the package of an order is. Takes the order id, not a shipment id. + Returns the shipment status, carrier, tracking number, last known location + and last scan time. Returns an error if the order has not shipped yet.","parameters":{"type":"object","properties":{"order_id":{"type":"integer","description":"The + order id the customer gives, for example 1042"}},"required":["order_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"get_customer","description":"Looks + up a customer by id. Returns their name, city and state. It does not return + contact details or orders; use search_orders to list a customer''s orders.","parameters":{"type":"object","properties":{"customer_id":{"type":"integer","description":"The + customer id, for example 68585"}},"required":["customer_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"search_orders","description":"Lists + a customer''s most recent orders, newest first, up to 10, with the real total + count. Each entry has the order id, status and dates. Use it when the customer + does not know their order id, then call get_order for the details of one order.","parameters":{"type":"object","properties":{"customer_id":{"type":"integer","description":"The + customer id, for example 68585"},"status":{"type":"string","description":"Only + orders with this status: created, approved, invoiced, processing, shipped, + delivered, canceled or unavailable"}},"required":["customer_id"],"additionalProperties":false}}}],"include_reasoning":false}' + headers: + Authorization: + - Bearer + User-Agent: + - Faraday v2.14.4 + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 12 Oct 2026 16:43:50 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Cache-Control: + - private, max-age=0, no-store, no-cache, must-revalidate + Server: + - cloudflare + Vary: + - Origin + X-Groq-Region: + - bom + X-Ratelimit-Limit-Requests: + - '1000' + X-Ratelimit-Limit-Tokens: + - '8000' + X-Ratelimit-Remaining-Requests: + - '996' + X-Ratelimit-Remaining-Tokens: + - '7226' + X-Ratelimit-Reset-Requests: + - 5m45.6s + X-Ratelimit-Reset-Tokens: + - 5.805s + X-Request-Id: + - req_01m4rfzgkhemts2f7jy51323xc + Set-Cookie: + - __cf_bm=5TDYd7uTMMS2FIJClUqs1ov0HnSKm5LVwyUbQ3eyFWA-1791823430.2059543-1.0.1.1-6sjaVi87pIaFgq8DiiNGQkIE20tT4IRHkBbtz0XO9ZtbEHMIoTUnIncCtJ_peSFKuK8tPFyPkQMROVfZ6Tdpd0_Boxg1DAbQJfTOPNsa8E_zMJQuNz9f0FZhZcNXA.TH; + HttpOnly; SameSite=None; Secure; Path=/; Domain=groq.com; Expires=Mon, 12 + Oct 2026 17:13:50 GMT + Via: + - 1.1 google + Strict-Transport-Security: + - max-age=15552000 + Cf-Cache-Status: + - DYNAMIC + Cf-Ray: + - a49790d6c9f15f7d-SIN + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: '{"id":"chatcmpl-452db371-7778-4e4c-8fd6-bcbcba44531f","object":"chat.completion","created":1791823430,"model":"openai/gpt-oss-120b","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"fc_046b58bc-ece1-406b-a865-17011e8bd307","type":"function","function":{"name":"get_shipment","arguments":"{\"order_id\":4242}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"queue_time":0.338176654,"prompt_tokens":440,"prompt_time":0.018495674,"completion_tokens":70,"completion_time":0.147806128,"total_tokens":510,"total_time":0.166301802,"completion_tokens_details":{"reasoning_tokens":40}},"usage_breakdown":null,"system_fingerprint":"fp_4200b3f836","x_groq":{"id":"req_01m4rfzgkhemts2f7jy51323xc","seed":1113271383},"service_tier":"on_demand"} + + ' + recorded_at: Mon, 12 Oct 2026 16:43:50 GMT +- request: + method: post + uri: https://api.groq.com/openai/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"openai/gpt-oss-120b","messages":[{"role":"user","content":"Where + is my order 4242?"},{"role":"assistant","tool_calls":[{"id":"fc_046b58bc-ece1-406b-a865-17011e8bd307","type":"function","function":{"name":"get_shipment","arguments":"{\"order_id\":4242}"}}]},{"role":"tool","tool_call_id":"fc_046b58bc-ece1-406b-a865-17011e8bd307","content":"{\"error\":\"Order + 4242 has no shipment\"}"}],"tools":[{"type":"function","function":{"name":"get_order","description":"Looks + up one order by its id. Returns its status, purchase, shipping and delivery + dates, and the items with name, category, price and whether they are final + sale.","parameters":{"type":"object","properties":{"order_id":{"type":"integer","description":"The + order id the customer gives, for example 1042"}},"required":["order_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"get_shipment","description":"Looks + up where the package of an order is. Takes the order id, not a shipment id. + Returns the shipment status, carrier, tracking number, last known location + and last scan time. Returns an error if the order has not shipped yet.","parameters":{"type":"object","properties":{"order_id":{"type":"integer","description":"The + order id the customer gives, for example 1042"}},"required":["order_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"get_customer","description":"Looks + up a customer by id. Returns their name, city and state. It does not return + contact details or orders; use search_orders to list a customer''s orders.","parameters":{"type":"object","properties":{"customer_id":{"type":"integer","description":"The + customer id, for example 68585"}},"required":["customer_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"search_orders","description":"Lists + a customer''s most recent orders, newest first, up to 10, with the real total + count. Each entry has the order id, status and dates. Use it when the customer + does not know their order id, then call get_order for the details of one order.","parameters":{"type":"object","properties":{"customer_id":{"type":"integer","description":"The + customer id, for example 68585"},"status":{"type":"string","description":"Only + orders with this status: created, approved, invoiced, processing, shipped, + delivered, canceled or unavailable"}},"required":["customer_id"],"additionalProperties":false}}}],"include_reasoning":false}' + headers: + Authorization: + - Bearer + User-Agent: + - Faraday v2.14.4 + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 12 Oct 2026 16:43:51 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Cache-Control: + - private, max-age=0, no-store, no-cache, must-revalidate + Server: + - cloudflare + Vary: + - Origin + X-Groq-Region: + - bom + X-Ratelimit-Limit-Requests: + - '1000' + X-Ratelimit-Limit-Tokens: + - '8000' + X-Ratelimit-Remaining-Requests: + - '995' + X-Ratelimit-Remaining-Tokens: + - '6817' + X-Ratelimit-Reset-Requests: + - 7m12s + X-Ratelimit-Reset-Tokens: + - 8.872s + X-Request-Id: + - req_01m4rfzhmjeh1rx76enst4x47q + Via: + - 1.1 google + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - __cf_bm=47aL_XnePEzuehLHwBi6Xc4sJP0O_1Vxv8Ir6GAHLWI-1791823431.2634177-1.0.1.1-fnDqVcFy_d9p1fQsl.y8apXHjFlLhcJe7LWBz0GFGXOoo18HC13t9lX7ClguFcPwhkls4.Xh8jWQqjL_aUQ.b2K90kBqtQRl9AMqsSKoJ3Yam7jQrLzR1MJnBKpmIUUk; + HttpOnly; SameSite=None; Secure; Path=/; Domain=groq.com; Expires=Mon, 12 + Oct 2026 17:13:51 GMT + Strict-Transport-Security: + - max-age=15552000 + Cf-Ray: + - a49790dd6a75411d-SIN + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: '{"id":"chatcmpl-c45fb532-b9a9-4795-8c70-5250157c8554","object":"chat.completion","created":1791823431,"model":"openai/gpt-oss-120b","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"fc_b84b9c4f-2b8b-48aa-ab1b-2b90f3afa68a","type":"function","function":{"name":"get_order","arguments":"{\"order_id\":4242}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"queue_time":0.392601226,"prompt_tokens":479,"prompt_time":0.024133861,"completion_tokens":56,"completion_time":0.117754548,"total_tokens":535,"total_time":0.141888409,"completion_tokens_details":{"reasoning_tokens":33}},"usage_breakdown":null,"system_fingerprint":"fp_9241e9962b","x_groq":{"id":"req_01m4rfzhmjeh1rx76enst4x47q","seed":374366644},"service_tier":"on_demand"} + + ' + recorded_at: Mon, 12 Oct 2026 16:43:51 GMT +- request: + method: post + uri: https://api.groq.com/openai/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"openai/gpt-oss-120b","messages":[{"role":"user","content":"Where + is my order 4242?"},{"role":"assistant","tool_calls":[{"id":"fc_046b58bc-ece1-406b-a865-17011e8bd307","type":"function","function":{"name":"get_shipment","arguments":"{\"order_id\":4242}"}}]},{"role":"tool","tool_call_id":"fc_046b58bc-ece1-406b-a865-17011e8bd307","content":"{\"error\":\"Order + 4242 has no shipment\"}"},{"role":"assistant","tool_calls":[{"id":"fc_b84b9c4f-2b8b-48aa-ab1b-2b90f3afa68a","type":"function","function":{"name":"get_order","arguments":"{\"order_id\":4242}"}}]},{"role":"tool","tool_call_id":"fc_b84b9c4f-2b8b-48aa-ab1b-2b90f3afa68a","content":"{\"id\":4242,\"status\":\"shipped\",\"purchased_at\":\"2026-09-20T00:00:00.000Z\",\"shipped_at\":\"2026-09-24T00:00:00.000Z\",\"delivered_at\":null,\"estimated_delivery_at\":\"2026-10-01T00:00:00.000Z\",\"items\":[]}"}],"tools":[{"type":"function","function":{"name":"get_order","description":"Looks + up one order by its id. Returns its status, purchase, shipping and delivery + dates, and the items with name, category, price and whether they are final + sale.","parameters":{"type":"object","properties":{"order_id":{"type":"integer","description":"The + order id the customer gives, for example 1042"}},"required":["order_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"get_shipment","description":"Looks + up where the package of an order is. Takes the order id, not a shipment id. + Returns the shipment status, carrier, tracking number, last known location + and last scan time. Returns an error if the order has not shipped yet.","parameters":{"type":"object","properties":{"order_id":{"type":"integer","description":"The + order id the customer gives, for example 1042"}},"required":["order_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"get_customer","description":"Looks + up a customer by id. Returns their name, city and state. It does not return + contact details or orders; use search_orders to list a customer''s orders.","parameters":{"type":"object","properties":{"customer_id":{"type":"integer","description":"The + customer id, for example 68585"}},"required":["customer_id"],"additionalProperties":false}}},{"type":"function","function":{"name":"search_orders","description":"Lists + a customer''s most recent orders, newest first, up to 10, with the real total + count. Each entry has the order id, status and dates. Use it when the customer + does not know their order id, then call get_order for the details of one order.","parameters":{"type":"object","properties":{"customer_id":{"type":"integer","description":"The + customer id, for example 68585"},"status":{"type":"string","description":"Only + orders with this status: created, approved, invoiced, processing, shipped, + delivered, canceled or unavailable"}},"required":["customer_id"],"additionalProperties":false}}}],"include_reasoning":false}' + headers: + Authorization: + - Bearer + User-Agent: + - Faraday v2.14.4 + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 12 Oct 2026 16:43:54 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Cache-Control: + - private, max-age=0, no-store, no-cache, must-revalidate + Server: + - cloudflare + Vary: + - Origin + X-Groq-Region: + - bom + X-Ratelimit-Limit-Requests: + - '1000' + X-Ratelimit-Limit-Tokens: + - '8000' + X-Ratelimit-Remaining-Requests: + - '994' + X-Ratelimit-Remaining-Tokens: + - '6297' + X-Ratelimit-Reset-Requests: + - 8m38.4s + X-Ratelimit-Reset-Tokens: + - 12.772s + X-Request-Id: + - req_01m4rfzjm5ejw8042k69bb68br + Set-Cookie: + - __cf_bm=HerrXfr8zTtyukGzjEgyxLcCILOiohilMbMauuCkjfw-1791823432.2810361-1.0.1.1-KGxnMIHQmPyMm2j6jzao7qCUDx9c8_UgvqdvvH83ryIoCiZUH6PNir4HwEmkBn4OKCfEU7nHAFqTM0IzBD5uv5gMQq_yRhwV45heMB04EPcOsQqwcEDdnCB5U_vSeSaH; + HttpOnly; SameSite=None; Secure; Path=/; Domain=groq.com; Expires=Mon, 12 + Oct 2026 17:13:54 GMT + Via: + - 1.1 google + Strict-Transport-Security: + - max-age=15552000 + Cf-Cache-Status: + - DYNAMIC + Cf-Ray: + - a49790e3bdcbd837-SIN + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + eyJpZCI6ImNoYXRjbXBsLTg2ZDE1N2NmLTA1ZDYtNDY4Yi04NjYwLWZhZjVjZmYyZWY5NyIsIm9iamVjdCI6ImNoYXQuY29tcGxldGlvbiIsImNyZWF0ZWQiOjE3OTE4MjM0MzQsIm1vZGVsIjoib3BlbmFpL2dwdC1vc3MtMTIwYiIsImNob2ljZXMiOlt7ImluZGV4IjowLCJtZXNzYWdlIjp7InJvbGUiOiJhc3Npc3RhbnQiLCJjb250ZW50IjoiWW91ciBvcmRlciAqKiM0MjQyKiogaGFzIGFscmVhZHkgYmVlbiBzaGlwcGVkIChpdCBsZWZ0IG91ciB3YXJlaG91c2Ugb27igK9TZXB0ZW1iZXLigK8yNCzigK8yMDI2KS4gV2hpbGUgYSBkZXRhaWxlZCB0cmFja2luZyByZWNvcmQgaXNu4oCZdCBhdmFpbGFibGUgeWV0LCB0aGUgc3lzdGVtIGVzdGltYXRlcyB0aGF0IGl0IHNob3VsZCBhcnJpdmUgYXJvdW5kICoqT2N0b2JlcuKArzEs4oCvMjAyNioqLlxuXG5JZiB5b3XigJlkIGxpa2UgYSB0cmFja2luZyBudW1iZXIgb3IgbW9yZSBwcmVjaXNlIGxvY2F0aW9uIHVwZGF0ZXMsIGp1c3QgbGV0IG1lIGtub3cgYW5kIEkgY2FuIGNoZWNrIGFnYWluIGluIGEgZGF5IG9yIHR3byB3aGVuIHRoZSBjYXJyaWVy4oCZcyBzY2FuIGRhdGEgaXMgYXZhaWxhYmxlLiJ9LCJsb2dwcm9icyI6bnVsbCwiZmluaXNoX3JlYXNvbiI6InN0b3AifV0sInVzYWdlIjp7InF1ZXVlX3RpbWUiOjAuMzM4MTAzMTE0LCJwcm9tcHRfdG9rZW5zIjo1ODUsInByb21wdF90aW1lIjoxLjcyMzcwNjkzNSwiY29tcGxldGlvbl90b2tlbnMiOjE3NSwiY29tcGxldGlvbl90aW1lIjowLjM2MjcxNjM2MywidG90YWxfdG9rZW5zIjo3NjAsInRvdGFsX3RpbWUiOjIuMDg2NDIzMjk4LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiY2FjaGVkX3Rva2VucyI6MjU2fSwiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6eyJyZWFzb25pbmdfdG9rZW5zIjo3Nn19LCJ1c2FnZV9icmVha2Rvd24iOm51bGwsInN5c3RlbV9maW5nZXJwcmludCI6ImZwXzQyMDBiM2Y4MzYiLCJ4X2dyb3EiOnsiaWQiOiJyZXFfMDFtNHJmemptNWVqdzgwNDJrNjliYjY4YnIiLCJzZWVkIjo4NDk4MjQwOTJ9LCJzZXJ2aWNlX3RpZXIiOiJvbl9kZW1hbmQifQo= + recorded_at: Mon, 12 Oct 2026 16:43:54 GMT +recorded_with: VCR 6.4.0 diff --git a/spec/support/vcr.rb b/spec/support/vcr.rb index 76cbab6..d8f7dea 100644 --- a/spec/support/vcr.rb +++ b/spec/support/vcr.rb @@ -14,6 +14,8 @@ config.filter_sensitive_data("") { RubyLLM.config.openai_api_key } end -# RubyLLM refuses to build a chat without a key, even when VCR only replays. -# This lets anyone run the specs from the cassettes without a Groq account. +# RubyLLM refuses to build a chat without a key, and LlmClient reads the key from ENV, +# even when VCR only replays. This lets anyone run the specs from the cassettes +# without a Groq account. RubyLLM.config.openai_api_key ||= "replay-only-key" +ENV["GROQ_API_KEY"] ||= "replay-only-key"