From 8e71c10f13b541f1ac748eda9e67a1c3b5d81583 Mon Sep 17 00:00:00 2001 From: Zane Chen <99708452+czl9707@users.noreply.github.com> Date: Tue, 19 May 2026 20:16:37 -0400 Subject: [PATCH] chore: respecting stop reason (#17) * chore: respecting stop reason * fix: add back template substitution * typo fix * increase timeout allow slower model --- 00-chat-loop/src/mybot/provider/llm/base.py | 2 ++ 01-tools/README.md | 18 +++++++++++----- 01-tools/src/mybot/core/agent.py | 21 ++++++++++++++----- 01-tools/src/mybot/provider/llm/__init__.py | 4 ++-- 01-tools/src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 02-skills/src/mybot/core/agent.py | 21 ++++++++++++++----- 02-skills/src/mybot/provider/llm/__init__.py | 4 ++-- 02-skills/src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 03-persistence/src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/provider/llm/__init__.py | 4 ++-- 03-persistence/src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 04-slash-commands/src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 05-compaction/src/mybot/core/agent.py | 21 ++++++++++++++----- 05-compaction/src/mybot/core/context_guard.py | 2 +- .../src/mybot/provider/llm/__init__.py | 4 ++-- 05-compaction/src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 06-web-tools/src/mybot/core/agent.py | 21 ++++++++++++++----- 06-web-tools/src/mybot/core/context_guard.py | 2 +- .../src/mybot/provider/llm/__init__.py | 4 ++-- 06-web-tools/src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 07-event-driven/src/mybot/cli/chat.py | 2 +- 07-event-driven/src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/core/context_guard.py | 2 +- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 08-config-hot-reload/src/mybot/cli/chat.py | 2 +- 08-config-hot-reload/src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/core/context_guard.py | 2 +- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 09-channels/src/mybot/cli/chat.py | 2 +- 09-channels/src/mybot/core/agent.py | 21 ++++++++++++++----- 09-channels/src/mybot/core/context_guard.py | 2 +- .../src/mybot/provider/llm/__init__.py | 4 ++-- 09-channels/src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 10-websocket/src/mybot/cli/chat.py | 2 +- 10-websocket/src/mybot/core/agent.py | 21 ++++++++++++++----- 10-websocket/src/mybot/core/context_guard.py | 2 +- .../src/mybot/provider/llm/__init__.py | 4 ++-- 10-websocket/src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 11-multi-agent-routing/src/mybot/cli/chat.py | 2 +- .../src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/core/context_guard.py | 2 +- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 12-cron-heartbeat/src/mybot/cli/chat.py | 2 +- 12-cron-heartbeat/src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/core/context_guard.py | 2 +- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 13-multi-layer-prompts/src/mybot/cli/chat.py | 2 +- .../src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/core/context_guard.py | 2 +- .../src/mybot/core/prompt_builder.py | 18 ++++++++++++++-- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 14-post-message-back/src/mybot/cli/chat.py | 2 +- 14-post-message-back/src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/core/context_guard.py | 2 +- .../src/mybot/core/prompt_builder.py | 18 ++++++++++++++-- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 15-agent-dispatch/src/mybot/cli/chat.py | 2 +- 15-agent-dispatch/src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/core/context_guard.py | 2 +- .../src/mybot/core/prompt_builder.py | 18 ++++++++++++++-- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 16-concurrency-control/src/mybot/cli/chat.py | 2 +- .../src/mybot/core/agent.py | 21 ++++++++++++++----- .../src/mybot/core/context_guard.py | 2 +- .../src/mybot/core/prompt_builder.py | 18 ++++++++++++++-- .../src/mybot/provider/llm/__init__.py | 4 ++-- .../src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 17-memory/README.md | 2 +- 17-memory/src/mybot/cli/chat.py | 2 +- 17-memory/src/mybot/core/agent.py | 21 ++++++++++++++----- 17-memory/src/mybot/core/context_guard.py | 2 +- 17-memory/src/mybot/core/prompt_builder.py | 18 ++++++++++++++-- 17-memory/src/mybot/provider/llm/__init__.py | 4 ++-- 17-memory/src/mybot/provider/llm/base.py | 21 ++++++++++++++++--- 83 files changed, 732 insertions(+), 210 deletions(-) diff --git a/00-chat-loop/src/mybot/provider/llm/base.py b/00-chat-loop/src/mybot/provider/llm/base.py index a54e51b..f5fd91a 100644 --- a/00-chat-loop/src/mybot/provider/llm/base.py +++ b/00-chat-loop/src/mybot/provider/llm/base.py @@ -50,6 +50,8 @@ class LLMProvider: "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: diff --git a/01-tools/README.md b/01-tools/README.md index 106071c..50fd547 100644 --- a/01-tools/README.md +++ b/01-tools/README.md @@ -19,7 +19,7 @@ Giving the agent the ability to actually *do* things, from chatting only to taki ## Key Components -- **Stop Reason**: Chat Loop can stop because of "end_turn" or "tool_use" +- **Stop Reason**: Chat loop branches on `stop_reason` — `"tool_calls"` to execute tools, `"stop"` for normal completion, `"length"` for truncated responses - **Tools**: Manages available tools and executes tool calls - **Tool Calling Loop**: Agent calls tools, adds results to history, continues conversation @@ -59,10 +59,11 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) assistant_msg: Message = { "role": "assistant", @@ -71,10 +72,17 @@ class AgentSession: } self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) + + break return content ``` diff --git a/01-tools/src/mybot/core/agent.py b/01-tools/src/mybot/core/agent.py index 4c36697..c91adfe 100644 --- a/01-tools/src/mybot/core/agent.py +++ b/01-tools/src/mybot/core/agent.py @@ -2,6 +2,7 @@ import asyncio import json +import logging import uuid from dataclasses import dataclass, field from datetime import datetime @@ -65,10 +66,11 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -86,12 +88,21 @@ class AgentSession: assistant_msg["tool_calls"] = tool_call_dicts self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/01-tools/src/mybot/provider/llm/__init__.py b/01-tools/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/01-tools/src/mybot/provider/llm/__init__.py +++ b/01-tools/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/01-tools/src/mybot/provider/llm/base.py b/01-tools/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/01-tools/src/mybot/provider/llm/base.py +++ b/01-tools/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/02-skills/src/mybot/core/agent.py b/02-skills/src/mybot/core/agent.py index 0bf57a6..95f6f99 100644 --- a/02-skills/src/mybot/core/agent.py +++ b/02-skills/src/mybot/core/agent.py @@ -2,6 +2,7 @@ import asyncio import json +import logging import uuid from dataclasses import dataclass, field from datetime import datetime @@ -81,10 +82,11 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -102,12 +104,21 @@ class AgentSession: assistant_msg["tool_calls"] = tool_call_dicts self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/02-skills/src/mybot/provider/llm/__init__.py b/02-skills/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/02-skills/src/mybot/provider/llm/__init__.py +++ b/02-skills/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/02-skills/src/mybot/provider/llm/base.py b/02-skills/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/02-skills/src/mybot/provider/llm/base.py +++ b/02-skills/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/03-persistence/src/mybot/core/agent.py b/03-persistence/src/mybot/core/agent.py index 25eb82f..f3246c8 100644 --- a/03-persistence/src/mybot/core/agent.py +++ b/03-persistence/src/mybot/core/agent.py @@ -2,6 +2,7 @@ import asyncio import json +import logging import uuid from dataclasses import dataclass, field from datetime import datetime @@ -83,10 +84,11 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -104,12 +106,21 @@ class AgentSession: assistant_msg["tool_calls"] = tool_call_dicts self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/03-persistence/src/mybot/provider/llm/__init__.py b/03-persistence/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/03-persistence/src/mybot/provider/llm/__init__.py +++ b/03-persistence/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/03-persistence/src/mybot/provider/llm/base.py b/03-persistence/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/03-persistence/src/mybot/provider/llm/base.py +++ b/03-persistence/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/04-slash-commands/src/mybot/core/agent.py b/04-slash-commands/src/mybot/core/agent.py index a8f721c..5edf66d 100644 --- a/04-slash-commands/src/mybot/core/agent.py +++ b/04-slash-commands/src/mybot/core/agent.py @@ -2,6 +2,7 @@ import asyncio import json +import logging import uuid from dataclasses import dataclass, field from datetime import datetime @@ -91,10 +92,11 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -112,12 +114,21 @@ class AgentSession: assistant_msg["tool_calls"] = tool_call_dicts self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/04-slash-commands/src/mybot/provider/llm/__init__.py b/04-slash-commands/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/04-slash-commands/src/mybot/provider/llm/__init__.py +++ b/04-slash-commands/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/04-slash-commands/src/mybot/provider/llm/base.py b/04-slash-commands/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/04-slash-commands/src/mybot/provider/llm/base.py +++ b/04-slash-commands/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/05-compaction/src/mybot/core/agent.py b/05-compaction/src/mybot/core/agent.py index 19ce21d..06308e1 100644 --- a/05-compaction/src/mybot/core/agent.py +++ b/05-compaction/src/mybot/core/agent.py @@ -2,6 +2,7 @@ import asyncio import json +import logging import uuid from dataclasses import dataclass, field from datetime import datetime @@ -104,13 +105,14 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -128,12 +130,21 @@ class AgentSession: assistant_msg["tool_calls"] = tool_call_dicts self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/05-compaction/src/mybot/core/context_guard.py b/05-compaction/src/mybot/core/context_guard.py index e12f4a2..529899e 100644 --- a/05-compaction/src/mybot/core/context_guard.py +++ b/05-compaction/src/mybot/core/context_guard.py @@ -134,7 +134,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/05-compaction/src/mybot/provider/llm/__init__.py b/05-compaction/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/05-compaction/src/mybot/provider/llm/__init__.py +++ b/05-compaction/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/05-compaction/src/mybot/provider/llm/base.py b/05-compaction/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/05-compaction/src/mybot/provider/llm/base.py +++ b/05-compaction/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/06-web-tools/src/mybot/core/agent.py b/06-web-tools/src/mybot/core/agent.py index 579190d..07e00fa 100644 --- a/06-web-tools/src/mybot/core/agent.py +++ b/06-web-tools/src/mybot/core/agent.py @@ -2,6 +2,7 @@ import asyncio import json +import logging import uuid from dataclasses import dataclass, field from datetime import datetime @@ -115,13 +116,14 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -139,12 +141,21 @@ class AgentSession: assistant_msg["tool_calls"] = tool_call_dicts self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/06-web-tools/src/mybot/core/context_guard.py b/06-web-tools/src/mybot/core/context_guard.py index e12f4a2..529899e 100644 --- a/06-web-tools/src/mybot/core/context_guard.py +++ b/06-web-tools/src/mybot/core/context_guard.py @@ -134,7 +134,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/06-web-tools/src/mybot/provider/llm/__init__.py b/06-web-tools/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/06-web-tools/src/mybot/provider/llm/__init__.py +++ b/06-web-tools/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/06-web-tools/src/mybot/provider/llm/base.py b/06-web-tools/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/06-web-tools/src/mybot/provider/llm/base.py +++ b/06-web-tools/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/07-event-driven/src/mybot/cli/chat.py b/07-event-driven/src/mybot/cli/chat.py index 7e8d6fa..bc2cc46 100644 --- a/07-event-driven/src/mybot/cli/chat.py +++ b/07-event-driven/src/mybot/cli/chat.py @@ -93,7 +93,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/07-event-driven/src/mybot/core/agent.py b/07-event-driven/src/mybot/core/agent.py index 33865ea..5669e52 100644 --- a/07-event-driven/src/mybot/core/agent.py +++ b/07-event-driven/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -157,11 +158,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -180,12 +182,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/07-event-driven/src/mybot/core/context_guard.py b/07-event-driven/src/mybot/core/context_guard.py index b73a83c..3be3c4f 100644 --- a/07-event-driven/src/mybot/core/context_guard.py +++ b/07-event-driven/src/mybot/core/context_guard.py @@ -137,7 +137,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/07-event-driven/src/mybot/provider/llm/__init__.py b/07-event-driven/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/07-event-driven/src/mybot/provider/llm/__init__.py +++ b/07-event-driven/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/07-event-driven/src/mybot/provider/llm/base.py b/07-event-driven/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/07-event-driven/src/mybot/provider/llm/base.py +++ b/07-event-driven/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/08-config-hot-reload/src/mybot/cli/chat.py b/08-config-hot-reload/src/mybot/cli/chat.py index cad9f33..d2f8757 100644 --- a/08-config-hot-reload/src/mybot/cli/chat.py +++ b/08-config-hot-reload/src/mybot/cli/chat.py @@ -96,7 +96,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/08-config-hot-reload/src/mybot/core/agent.py b/08-config-hot-reload/src/mybot/core/agent.py index 33865ea..5669e52 100644 --- a/08-config-hot-reload/src/mybot/core/agent.py +++ b/08-config-hot-reload/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -157,11 +158,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -180,12 +182,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/08-config-hot-reload/src/mybot/core/context_guard.py b/08-config-hot-reload/src/mybot/core/context_guard.py index b73a83c..3be3c4f 100644 --- a/08-config-hot-reload/src/mybot/core/context_guard.py +++ b/08-config-hot-reload/src/mybot/core/context_guard.py @@ -137,7 +137,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/08-config-hot-reload/src/mybot/provider/llm/__init__.py b/08-config-hot-reload/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/08-config-hot-reload/src/mybot/provider/llm/__init__.py +++ b/08-config-hot-reload/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/08-config-hot-reload/src/mybot/provider/llm/base.py b/08-config-hot-reload/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/08-config-hot-reload/src/mybot/provider/llm/base.py +++ b/08-config-hot-reload/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/09-channels/src/mybot/cli/chat.py b/09-channels/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/09-channels/src/mybot/cli/chat.py +++ b/09-channels/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/09-channels/src/mybot/core/agent.py b/09-channels/src/mybot/core/agent.py index 01ed2a9..cc0ce97 100644 --- a/09-channels/src/mybot/core/agent.py +++ b/09-channels/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -168,11 +169,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -191,12 +193,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/09-channels/src/mybot/core/context_guard.py b/09-channels/src/mybot/core/context_guard.py index dde78da..702344b 100644 --- a/09-channels/src/mybot/core/context_guard.py +++ b/09-channels/src/mybot/core/context_guard.py @@ -153,7 +153,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/09-channels/src/mybot/provider/llm/__init__.py b/09-channels/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/09-channels/src/mybot/provider/llm/__init__.py +++ b/09-channels/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/09-channels/src/mybot/provider/llm/base.py b/09-channels/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/09-channels/src/mybot/provider/llm/base.py +++ b/09-channels/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/10-websocket/src/mybot/cli/chat.py b/10-websocket/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/10-websocket/src/mybot/cli/chat.py +++ b/10-websocket/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/10-websocket/src/mybot/core/agent.py b/10-websocket/src/mybot/core/agent.py index 01ed2a9..cc0ce97 100644 --- a/10-websocket/src/mybot/core/agent.py +++ b/10-websocket/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -168,11 +169,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -191,12 +193,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/10-websocket/src/mybot/core/context_guard.py b/10-websocket/src/mybot/core/context_guard.py index dde78da..702344b 100644 --- a/10-websocket/src/mybot/core/context_guard.py +++ b/10-websocket/src/mybot/core/context_guard.py @@ -153,7 +153,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/10-websocket/src/mybot/provider/llm/__init__.py b/10-websocket/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/10-websocket/src/mybot/provider/llm/__init__.py +++ b/10-websocket/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/10-websocket/src/mybot/provider/llm/base.py b/10-websocket/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/10-websocket/src/mybot/provider/llm/base.py +++ b/10-websocket/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/11-multi-agent-routing/src/mybot/cli/chat.py b/11-multi-agent-routing/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/11-multi-agent-routing/src/mybot/cli/chat.py +++ b/11-multi-agent-routing/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/11-multi-agent-routing/src/mybot/core/agent.py b/11-multi-agent-routing/src/mybot/core/agent.py index 01ed2a9..cc0ce97 100644 --- a/11-multi-agent-routing/src/mybot/core/agent.py +++ b/11-multi-agent-routing/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -168,11 +169,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -191,12 +193,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/11-multi-agent-routing/src/mybot/core/context_guard.py b/11-multi-agent-routing/src/mybot/core/context_guard.py index 8c24571..e7aa46d 100644 --- a/11-multi-agent-routing/src/mybot/core/context_guard.py +++ b/11-multi-agent-routing/src/mybot/core/context_guard.py @@ -154,7 +154,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/11-multi-agent-routing/src/mybot/provider/llm/__init__.py b/11-multi-agent-routing/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/11-multi-agent-routing/src/mybot/provider/llm/__init__.py +++ b/11-multi-agent-routing/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/11-multi-agent-routing/src/mybot/provider/llm/base.py b/11-multi-agent-routing/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/11-multi-agent-routing/src/mybot/provider/llm/base.py +++ b/11-multi-agent-routing/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/12-cron-heartbeat/src/mybot/cli/chat.py b/12-cron-heartbeat/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/12-cron-heartbeat/src/mybot/cli/chat.py +++ b/12-cron-heartbeat/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/12-cron-heartbeat/src/mybot/core/agent.py b/12-cron-heartbeat/src/mybot/core/agent.py index 01ed2a9..cc0ce97 100644 --- a/12-cron-heartbeat/src/mybot/core/agent.py +++ b/12-cron-heartbeat/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -168,11 +169,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -191,12 +193,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/12-cron-heartbeat/src/mybot/core/context_guard.py b/12-cron-heartbeat/src/mybot/core/context_guard.py index b8b802c..c1f75df 100644 --- a/12-cron-heartbeat/src/mybot/core/context_guard.py +++ b/12-cron-heartbeat/src/mybot/core/context_guard.py @@ -154,7 +154,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/12-cron-heartbeat/src/mybot/provider/llm/__init__.py b/12-cron-heartbeat/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/12-cron-heartbeat/src/mybot/provider/llm/__init__.py +++ b/12-cron-heartbeat/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/12-cron-heartbeat/src/mybot/provider/llm/base.py b/12-cron-heartbeat/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/12-cron-heartbeat/src/mybot/provider/llm/base.py +++ b/12-cron-heartbeat/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/13-multi-layer-prompts/src/mybot/cli/chat.py b/13-multi-layer-prompts/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/13-multi-layer-prompts/src/mybot/cli/chat.py +++ b/13-multi-layer-prompts/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/13-multi-layer-prompts/src/mybot/core/agent.py b/13-multi-layer-prompts/src/mybot/core/agent.py index 01ed2a9..cc0ce97 100644 --- a/13-multi-layer-prompts/src/mybot/core/agent.py +++ b/13-multi-layer-prompts/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -168,11 +169,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -191,12 +193,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/13-multi-layer-prompts/src/mybot/core/context_guard.py b/13-multi-layer-prompts/src/mybot/core/context_guard.py index b8b802c..c1f75df 100644 --- a/13-multi-layer-prompts/src/mybot/core/context_guard.py +++ b/13-multi-layer-prompts/src/mybot/core/context_guard.py @@ -154,7 +154,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/13-multi-layer-prompts/src/mybot/core/prompt_builder.py b/13-multi-layer-prompts/src/mybot/core/prompt_builder.py index 36a34ad..55a8cf6 100644 --- a/13-multi-layer-prompts/src/mybot/core/prompt_builder.py +++ b/13-multi-layer-prompts/src/mybot/core/prompt_builder.py @@ -45,17 +45,31 @@ class PromptBuilder: return "\n\n".join(layers) + def _substitute_paths(self, text: str) -> str: + """Replace {{placeholder}} tokens with resolved config paths.""" + cfg = self.context.config + replacements = { + "{{workspace}}": str(cfg.workspace), + "{{skills_path}}": str(cfg.skills_path), + "{{crons_path}}": str(cfg.crons_path), + "{{memories_path}}": str(cfg.memories_path), + "{{agents_path}}": str(cfg.agents_path), + } + for placeholder, value in replacements.items(): + text = text.replace(placeholder, value) + return text + def _load_bootstrap_context(self) -> str: """Load BOOTSTRAP.md + AGENTS.md + cron list.""" parts = [] bootstrap_path = self.context.config.workspace / "BOOTSTRAP.md" if bootstrap_path.exists(): - parts.append(bootstrap_path.read_text().strip()) + parts.append(self._substitute_paths(bootstrap_path.read_text().strip())) agents_path = self.context.config.workspace / "AGENTS.md" if agents_path.exists(): - parts.append(agents_path.read_text().strip()) + parts.append(self._substitute_paths(agents_path.read_text().strip())) # Dynamic cron list cron_list = self._format_cron_list() diff --git a/13-multi-layer-prompts/src/mybot/provider/llm/__init__.py b/13-multi-layer-prompts/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/13-multi-layer-prompts/src/mybot/provider/llm/__init__.py +++ b/13-multi-layer-prompts/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/13-multi-layer-prompts/src/mybot/provider/llm/base.py b/13-multi-layer-prompts/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/13-multi-layer-prompts/src/mybot/provider/llm/base.py +++ b/13-multi-layer-prompts/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/14-post-message-back/src/mybot/cli/chat.py b/14-post-message-back/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/14-post-message-back/src/mybot/cli/chat.py +++ b/14-post-message-back/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/14-post-message-back/src/mybot/core/agent.py b/14-post-message-back/src/mybot/core/agent.py index 5eba1d7..9f31289 100644 --- a/14-post-message-back/src/mybot/core/agent.py +++ b/14-post-message-back/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -177,11 +178,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -200,12 +202,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/14-post-message-back/src/mybot/core/context_guard.py b/14-post-message-back/src/mybot/core/context_guard.py index b8b802c..c1f75df 100644 --- a/14-post-message-back/src/mybot/core/context_guard.py +++ b/14-post-message-back/src/mybot/core/context_guard.py @@ -154,7 +154,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/14-post-message-back/src/mybot/core/prompt_builder.py b/14-post-message-back/src/mybot/core/prompt_builder.py index 36a34ad..55a8cf6 100644 --- a/14-post-message-back/src/mybot/core/prompt_builder.py +++ b/14-post-message-back/src/mybot/core/prompt_builder.py @@ -45,17 +45,31 @@ class PromptBuilder: return "\n\n".join(layers) + def _substitute_paths(self, text: str) -> str: + """Replace {{placeholder}} tokens with resolved config paths.""" + cfg = self.context.config + replacements = { + "{{workspace}}": str(cfg.workspace), + "{{skills_path}}": str(cfg.skills_path), + "{{crons_path}}": str(cfg.crons_path), + "{{memories_path}}": str(cfg.memories_path), + "{{agents_path}}": str(cfg.agents_path), + } + for placeholder, value in replacements.items(): + text = text.replace(placeholder, value) + return text + def _load_bootstrap_context(self) -> str: """Load BOOTSTRAP.md + AGENTS.md + cron list.""" parts = [] bootstrap_path = self.context.config.workspace / "BOOTSTRAP.md" if bootstrap_path.exists(): - parts.append(bootstrap_path.read_text().strip()) + parts.append(self._substitute_paths(bootstrap_path.read_text().strip())) agents_path = self.context.config.workspace / "AGENTS.md" if agents_path.exists(): - parts.append(agents_path.read_text().strip()) + parts.append(self._substitute_paths(agents_path.read_text().strip())) # Dynamic cron list cron_list = self._format_cron_list() diff --git a/14-post-message-back/src/mybot/provider/llm/__init__.py b/14-post-message-back/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/14-post-message-back/src/mybot/provider/llm/__init__.py +++ b/14-post-message-back/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/14-post-message-back/src/mybot/provider/llm/base.py b/14-post-message-back/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/14-post-message-back/src/mybot/provider/llm/base.py +++ b/14-post-message-back/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/15-agent-dispatch/src/mybot/cli/chat.py b/15-agent-dispatch/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/15-agent-dispatch/src/mybot/cli/chat.py +++ b/15-agent-dispatch/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/15-agent-dispatch/src/mybot/core/agent.py b/15-agent-dispatch/src/mybot/core/agent.py index 21e07a8..4f96be4 100644 --- a/15-agent-dispatch/src/mybot/core/agent.py +++ b/15-agent-dispatch/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -185,11 +186,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -208,12 +210,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/15-agent-dispatch/src/mybot/core/context_guard.py b/15-agent-dispatch/src/mybot/core/context_guard.py index b8b802c..c1f75df 100644 --- a/15-agent-dispatch/src/mybot/core/context_guard.py +++ b/15-agent-dispatch/src/mybot/core/context_guard.py @@ -154,7 +154,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/15-agent-dispatch/src/mybot/core/prompt_builder.py b/15-agent-dispatch/src/mybot/core/prompt_builder.py index 36a34ad..55a8cf6 100644 --- a/15-agent-dispatch/src/mybot/core/prompt_builder.py +++ b/15-agent-dispatch/src/mybot/core/prompt_builder.py @@ -45,17 +45,31 @@ class PromptBuilder: return "\n\n".join(layers) + def _substitute_paths(self, text: str) -> str: + """Replace {{placeholder}} tokens with resolved config paths.""" + cfg = self.context.config + replacements = { + "{{workspace}}": str(cfg.workspace), + "{{skills_path}}": str(cfg.skills_path), + "{{crons_path}}": str(cfg.crons_path), + "{{memories_path}}": str(cfg.memories_path), + "{{agents_path}}": str(cfg.agents_path), + } + for placeholder, value in replacements.items(): + text = text.replace(placeholder, value) + return text + def _load_bootstrap_context(self) -> str: """Load BOOTSTRAP.md + AGENTS.md + cron list.""" parts = [] bootstrap_path = self.context.config.workspace / "BOOTSTRAP.md" if bootstrap_path.exists(): - parts.append(bootstrap_path.read_text().strip()) + parts.append(self._substitute_paths(bootstrap_path.read_text().strip())) agents_path = self.context.config.workspace / "AGENTS.md" if agents_path.exists(): - parts.append(agents_path.read_text().strip()) + parts.append(self._substitute_paths(agents_path.read_text().strip())) # Dynamic cron list cron_list = self._format_cron_list() diff --git a/15-agent-dispatch/src/mybot/provider/llm/__init__.py b/15-agent-dispatch/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/15-agent-dispatch/src/mybot/provider/llm/__init__.py +++ b/15-agent-dispatch/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/15-agent-dispatch/src/mybot/provider/llm/base.py b/15-agent-dispatch/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/15-agent-dispatch/src/mybot/provider/llm/base.py +++ b/15-agent-dispatch/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/16-concurrency-control/src/mybot/cli/chat.py b/16-concurrency-control/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/16-concurrency-control/src/mybot/cli/chat.py +++ b/16-concurrency-control/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/16-concurrency-control/src/mybot/core/agent.py b/16-concurrency-control/src/mybot/core/agent.py index 21e07a8..4f96be4 100644 --- a/16-concurrency-control/src/mybot/core/agent.py +++ b/16-concurrency-control/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -185,11 +186,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -208,12 +210,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/16-concurrency-control/src/mybot/core/context_guard.py b/16-concurrency-control/src/mybot/core/context_guard.py index b8b802c..c1f75df 100644 --- a/16-concurrency-control/src/mybot/core/context_guard.py +++ b/16-concurrency-control/src/mybot/core/context_guard.py @@ -154,7 +154,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/16-concurrency-control/src/mybot/core/prompt_builder.py b/16-concurrency-control/src/mybot/core/prompt_builder.py index 36a34ad..55a8cf6 100644 --- a/16-concurrency-control/src/mybot/core/prompt_builder.py +++ b/16-concurrency-control/src/mybot/core/prompt_builder.py @@ -45,17 +45,31 @@ class PromptBuilder: return "\n\n".join(layers) + def _substitute_paths(self, text: str) -> str: + """Replace {{placeholder}} tokens with resolved config paths.""" + cfg = self.context.config + replacements = { + "{{workspace}}": str(cfg.workspace), + "{{skills_path}}": str(cfg.skills_path), + "{{crons_path}}": str(cfg.crons_path), + "{{memories_path}}": str(cfg.memories_path), + "{{agents_path}}": str(cfg.agents_path), + } + for placeholder, value in replacements.items(): + text = text.replace(placeholder, value) + return text + def _load_bootstrap_context(self) -> str: """Load BOOTSTRAP.md + AGENTS.md + cron list.""" parts = [] bootstrap_path = self.context.config.workspace / "BOOTSTRAP.md" if bootstrap_path.exists(): - parts.append(bootstrap_path.read_text().strip()) + parts.append(self._substitute_paths(bootstrap_path.read_text().strip())) agents_path = self.context.config.workspace / "AGENTS.md" if agents_path.exists(): - parts.append(agents_path.read_text().strip()) + parts.append(self._substitute_paths(agents_path.read_text().strip())) # Dynamic cron list cron_list = self._format_cron_list() diff --git a/16-concurrency-control/src/mybot/provider/llm/__init__.py b/16-concurrency-control/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/16-concurrency-control/src/mybot/provider/llm/__init__.py +++ b/16-concurrency-control/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/16-concurrency-control/src/mybot/provider/llm/base.py b/16-concurrency-control/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/16-concurrency-control/src/mybot/provider/llm/base.py +++ b/16-concurrency-control/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, ) diff --git a/17-memory/README.md b/17-memory/README.md index d8b050a..239cb33 100644 --- a/17-memory/README.md +++ b/17-memory/README.md @@ -31,7 +31,7 @@ cookie: Yes, . cd 17-memory uv run my-bot chat -# You: Remember that I my name is Zane +# You: Remember that my name is Zane # Pickle: Got it! I've saved that preference. uv run my-bot chat diff --git a/17-memory/src/mybot/cli/chat.py b/17-memory/src/mybot/cli/chat.py index 77dcab9..28332e0 100644 --- a/17-memory/src/mybot/cli/chat.py +++ b/17-memory/src/mybot/cli/chat.py @@ -100,7 +100,7 @@ class ChatLoop: try: response = await asyncio.wait_for( - self.response_queue.get(), timeout=60.0 + self.response_queue.get(), timeout=120.0 ) self.display_agent_response(response.content) diff --git a/17-memory/src/mybot/core/agent.py b/17-memory/src/mybot/core/agent.py index 21e07a8..4f96be4 100644 --- a/17-memory/src/mybot/core/agent.py +++ b/17-memory/src/mybot/core/agent.py @@ -1,5 +1,6 @@ import uuid import json +import logging import asyncio from dataclasses import dataclass, field from datetime import datetime @@ -185,11 +186,12 @@ class AgentSession: self.state.add_message(user_msg) tool_schemas = self.tools.get_tool_schemas() + logger = logging.getLogger(__name__) while True: messages = self.state.build_messages() self.state = await self.context_guard.check_and_compact(self.state) - content, tool_calls = await self.agent.llm.chat(messages, tool_schemas) + content, tool_calls, stop_reason = await self.agent.llm.chat(messages, tool_schemas) tool_call_dicts: list[ChatCompletionMessageToolCallParam] = [ { @@ -208,12 +210,21 @@ class AgentSession: self.state.add_message(assistant_msg) - if not tool_calls: - break + if stop_reason == "tool_calls": + await self._handle_tool_calls(tool_calls) + continue - await self._handle_tool_calls(tool_calls) + if stop_reason == "length": + logger.warning( + "LLM response truncated (max_tokens reached), " + "returning partial response" + ) - continue + if stop_reason == "content_filter": + logger.warning("LLM response filtered by content filter") + return content if content else "I'm unable to respond to that request." + + break return content diff --git a/17-memory/src/mybot/core/context_guard.py b/17-memory/src/mybot/core/context_guard.py index b8b802c..c1f75df 100644 --- a/17-memory/src/mybot/core/context_guard.py +++ b/17-memory/src/mybot/core/context_guard.py @@ -154,7 +154,7 @@ class ContextGuard: summary_prompt = COMPACT_PROMPT.format(conversation=old_text) - response, _ = await state.agent.llm.chat( + response, _, _ = await state.agent.llm.chat( [{"role": "user", "content": summary_prompt}], [], # No tools needed ) diff --git a/17-memory/src/mybot/core/prompt_builder.py b/17-memory/src/mybot/core/prompt_builder.py index 36a34ad..55a8cf6 100644 --- a/17-memory/src/mybot/core/prompt_builder.py +++ b/17-memory/src/mybot/core/prompt_builder.py @@ -45,17 +45,31 @@ class PromptBuilder: return "\n\n".join(layers) + def _substitute_paths(self, text: str) -> str: + """Replace {{placeholder}} tokens with resolved config paths.""" + cfg = self.context.config + replacements = { + "{{workspace}}": str(cfg.workspace), + "{{skills_path}}": str(cfg.skills_path), + "{{crons_path}}": str(cfg.crons_path), + "{{memories_path}}": str(cfg.memories_path), + "{{agents_path}}": str(cfg.agents_path), + } + for placeholder, value in replacements.items(): + text = text.replace(placeholder, value) + return text + def _load_bootstrap_context(self) -> str: """Load BOOTSTRAP.md + AGENTS.md + cron list.""" parts = [] bootstrap_path = self.context.config.workspace / "BOOTSTRAP.md" if bootstrap_path.exists(): - parts.append(bootstrap_path.read_text().strip()) + parts.append(self._substitute_paths(bootstrap_path.read_text().strip())) agents_path = self.context.config.workspace / "AGENTS.md" if agents_path.exists(): - parts.append(agents_path.read_text().strip()) + parts.append(self._substitute_paths(agents_path.read_text().strip())) # Dynamic cron list cron_list = self._format_cron_list() diff --git a/17-memory/src/mybot/provider/llm/__init__.py b/17-memory/src/mybot/provider/llm/__init__.py index c1dfcd6..7a79b9a 100644 --- a/17-memory/src/mybot/provider/llm/__init__.py +++ b/17-memory/src/mybot/provider/llm/__init__.py @@ -1,5 +1,5 @@ """LLM provider abstraction.""" -from .base import LLMProvider, LLMToolCall +from .base import LLMProvider, LLMToolCall, StopReason -__all__ = ["LLMProvider", "LLMToolCall"] +__all__ = ["LLMProvider", "LLMToolCall", "StopReason"] diff --git a/17-memory/src/mybot/provider/llm/base.py b/17-memory/src/mybot/provider/llm/base.py index 0f3c657..0a7afb2 100644 --- a/17-memory/src/mybot/provider/llm/base.py +++ b/17-memory/src/mybot/provider/llm/base.py @@ -5,10 +5,13 @@ from typing import Any, Optional, cast from litellm import acompletion, Choices, TYPE_CHECKING from litellm.types.completion import ChatCompletionMessageParam as Message +from litellm.types.utils import OpenAIChatCompletionFinishReason if TYPE_CHECKING: from mybot.utils.config import LLMConfig +StopReason = OpenAIChatCompletionFinishReason + @dataclass class LLMToolCall: @@ -55,12 +58,21 @@ class LLMProvider: messages: list[Message], tools: Optional[list[dict[str, Any]]] = None, **kwargs: Any, - ) -> tuple[str, list[LLMToolCall]]: - """Default implementation using litellm. Subclasses can override.""" + ) -> tuple[str, list[LLMToolCall], StopReason]: + """Send a chat request to the LLM. + + Default implementation using litellm. Subclasses can override + if provider-specific behavior is needed. + + Returns: + Tuple of (content, tool_calls, stop_reason) + """ request_kwargs: dict[str, Any] = { "model": self.model, "messages": messages, "api_key": self.api_key, + "temperature": self.temperature, + "max_tokens": self.max_tokens, } if self.api_base: @@ -71,7 +83,9 @@ class LLMProvider: response = await acompletion(**request_kwargs) - message = cast(Choices, response.choices[0]).message + choice = cast(Choices, response.choices[0]) + message = choice.message + stop_reason = choice.finish_reason return ( message.content or "", @@ -83,4 +97,5 @@ class LLMProvider: ) for tc in (message.tool_calls or []) ], + stop_reason, )