Skip to content

Commit 8eeca6e

Browse files
dirkbrndwillemcdejongh
dirkbrnd
and
willemcdejongh
authored
fix: Handle None prompt with media (#3410)
## Summary If no message is provided, but images/audio/video/files is provided, there should still be a user message ## Type of change - [x] Bug fix - [ ] New feature - [ ] Breaking change - [ ] Improvement - [ ] Model update - [ ] Other: --- ## Checklist - [ ] Code complies with style guidelines - [ ] Ran format/validation scripts (`./scripts/format.sh` and `./scripts/validate.sh`) - [ ] Self-review completed - [ ] Documentation updated (comments, docstrings) - [ ] Examples and guides: Relevant cookbook examples have been included or updated (if applicable) - [ ] Tested in clean environment - [ ] Tests added/updated (if applicable) --- ## Additional Notes Add any important context (deployment instructions, screenshots, security considerations, etc.) --------- Co-authored-by: Willem Carel de Jongh <willemcarel@gmail.com>
1 parent cc65122 commit 8eeca6e

File tree

5 files changed

+186
-9
lines changed

5 files changed

+186
-9
lines changed

‎libs/agno/agno/agent/agent.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4347,9 +4347,21 @@ def get_user_message(
43474347
)
43484348

43494349
# 3. Build the default user message for the Agent
4350-
# If the message is None, return None
43514350
if message is None:
4352-
return None
4351+
# If we have any media, return a message with empty content
4352+
if images is not None or audio is not None or videos is not None or files is not None:
4353+
return Message(
4354+
role=self.user_message_role,
4355+
content="",
4356+
images=images,
4357+
audio=audio,
4358+
videos=videos,
4359+
files=files,
4360+
**kwargs,
4361+
)
4362+
else:
4363+
# If the message is None, return None
4364+
return None
43534365

43544366
user_msg_content = message
43554367
# Format the message with the session state variables

‎libs/agno/agno/team/team.py‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -522,7 +522,7 @@ def is_streamable(self) -> bool:
522522
@overload
523523
def run(
524524
self,
525-
message: Union[str, List, Dict, Message],
525+
message: Optional[Union[str, List, Dict, Message]] = None,
526526
*,
527527
stream: Literal[False] = False,
528528
stream_intermediate_steps: bool = False,
@@ -539,7 +539,7 @@ def run(
539539
@overload
540540
def run(
541541
self,
542-
message: Union[str, List, Dict, Message],
542+
message: Optional[Union[str, List, Dict, Message]] = None,
543543
*,
544544
stream: Literal[True] = True,
545545
stream_intermediate_steps: bool = False,
@@ -555,7 +555,7 @@ def run(
555555

556556
def run(
557557
self,
558-
message: Union[str, List, Dict, Message],
558+
message: Optional[Union[str, List, Dict, Message]] = None,
559559
*,
560560
stream: bool = False,
561561
stream_intermediate_steps: bool = False,
@@ -998,7 +998,7 @@ def _run_stream(
998998
@overload
999999
async def arun(
10001000
self,
1001-
message: Union[str, List, Dict, Message],
1001+
message: Optional[Union[str, List, Dict, Message]] = None,
10021002
*,
10031003
stream: Literal[False] = False,
10041004
stream_intermediate_steps: bool = False,
@@ -1015,7 +1015,7 @@ async def arun(
10151015
@overload
10161016
async def arun(
10171017
self,
1018-
message: Union[str, List, Dict, Message],
1018+
message: Optional[Union[str, List, Dict, Message]] = None,
10191019
*,
10201020
stream: Literal[True] = True,
10211021
stream_intermediate_steps: bool = False,
@@ -1031,7 +1031,7 @@ async def arun(
10311031

10321032
async def arun(
10331033
self,
1034-
message: Union[str, List, Dict, Message],
1034+
message: Optional[Union[str, List, Dict, Message]] = None,
10351035
*,
10361036
stream: bool = False,
10371037
stream_intermediate_steps: bool = False,
@@ -4699,7 +4699,7 @@ def get_run_messages(
46994699

47004700
def _get_user_message(
47014701
self,
4702-
message: Union[str, List, Dict, Message],
4702+
message: Optional[Union[str, List, Dict, Message]] = None,
47034703
audio: Optional[Sequence[Audio]] = None,
47044704
images: Optional[Sequence[Image]] = None,
47054705
videos: Optional[Sequence[Video]] = None,
@@ -4737,6 +4737,21 @@ def _get_user_message(
47374737
files=files,
47384738
**kwargs,
47394739
)
4740+
elif message is None:
4741+
# If we have any media, return a message with empty content
4742+
if images is not None or audio is not None or videos is not None or files is not None:
4743+
return Message(
4744+
role="user",
4745+
content="",
4746+
images=images,
4747+
audio=audio,
4748+
videos=videos,
4749+
files=files,
4750+
**kwargs,
4751+
)
4752+
else:
4753+
# If the message is None, return None
4754+
return None
47404755

47414756
# 3.2 If message is provided as a Message, use it directly
47424757
elif isinstance(message, Message):

‎libs/agno/tests/integration/agent/test_agent_multimodal.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
2+
3+
from agno.agent.agent import Agent
4+
from agno.media import Image
5+
from agno.models.openai.chat import OpenAIChat
6+
7+
8+
def test_agent_image_input(agent_storage):
9+
agent = Agent(
10+
model=OpenAIChat(id="gpt-4o-mini"),
11+
markdown=True,
12+
storage=agent_storage,
13+
)
14+
15+
response = agent.run(
16+
"Tell me about this image and give me the latest news about it.",
17+
images=[
18+
Image(
19+
url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg"
20+
)
21+
],
22+
)
23+
assert response.content is not None
24+
25+
session_in_db = agent_storage.read(response.session_id)
26+
assert session_in_db is not None
27+
assert session_in_db.memory["runs"] is not None
28+
assert len(session_in_db.memory["runs"]) == 1
29+
assert session_in_db.memory["runs"][0]["messages"] is not None
30+
assert len(session_in_db.memory["runs"][0]["messages"]) == 3
31+
assert session_in_db.memory["runs"][0]["messages"][1]["role"] == "user"
32+
assert session_in_db.memory["runs"][0]["messages"][2]["role"] == "assistant"
33+
assert session_in_db.memory["runs"][0]["messages"][1]["images"] is not None
34+
assert session_in_db.memory["runs"][0]["messages"][1]["images"][0]["url"] == "https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg"
35+
36+
37+
def test_agent_image_input_no_prompt(agent_storage):
38+
agent = Agent(
39+
model=OpenAIChat(id="gpt-4o-mini"),
40+
markdown=True,
41+
storage=agent_storage,
42+
)
43+
44+
response = agent.run(
45+
images=[
46+
Image(
47+
url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg"
48+
)
49+
],
50+
)
51+
assert response.content is not None
52+
53+
session_in_db = agent_storage.read(response.session_id)
54+
assert session_in_db is not None
55+
assert session_in_db.memory["runs"] is not None
56+
assert len(session_in_db.memory["runs"]) == 1
57+
assert session_in_db.memory["runs"][0]["messages"] is not None
58+
assert len(session_in_db.memory["runs"][0]["messages"]) == 3
59+
assert session_in_db.memory["runs"][0]["messages"][1]["role"] == "user"
60+
assert session_in_db.memory["runs"][0]["messages"][2]["role"] == "assistant"
61+
assert session_in_db.memory["runs"][0]["messages"][1]["images"] is not None
62+
assert session_in_db.memory["runs"][0]["messages"][1]["images"][0]["url"] == "https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg"

‎libs/agno/tests/integration/agent/conftest.py‎ renamed to ‎libs/agno/tests/integration/conftest.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,17 @@ def agent_storage(temp_storage_db_file):
4646
return storage
4747

4848

49+
@pytest.fixture
50+
def team_storage(temp_storage_db_file):
51+
"""Create a SQLite storage for team sessions."""
52+
# Use a unique table name for each test run
53+
table_name = f"team_sessions_{uuid.uuid4().hex[:8]}"
54+
storage = SqliteStorage(table_name=table_name, db_file=temp_storage_db_file, mode="team")
55+
storage.create()
56+
return storage
57+
58+
59+
4960
@pytest.fixture
5061
def memory_db(temp_memory_db_file):
5162
"""Create a SQLite memory database for testing."""

‎libs/agno/tests/integration/teams/test_team_multimodal.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
2+
3+
from agno.agent.agent import Agent
4+
from agno.media import Image
5+
from agno.models.openai.chat import OpenAIChat
6+
from agno.team.team import Team
7+
8+
9+
def test_team_image_input(team_storage, agent_storage):
10+
image_analyst = Agent(
11+
name="Image Analyst",
12+
role="Analyze images and provide insights.",
13+
model=OpenAIChat(id="gpt-4o-mini"),
14+
markdown=True,
15+
storage=agent_storage,
16+
)
17+
18+
team = Team(
19+
model=OpenAIChat(id="gpt-4o-mini"),
20+
members=[image_analyst],
21+
name="Team",
22+
storage=team_storage,
23+
)
24+
25+
response = team.run(
26+
"Tell me about this image and give me the latest news about it.",
27+
images=[
28+
Image(
29+
url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg"
30+
)
31+
],
32+
)
33+
assert response.content is not None
34+
35+
session_in_db = team_storage.read(response.session_id)
36+
assert session_in_db is not None
37+
assert session_in_db.memory["runs"] is not None
38+
assert len(session_in_db.memory["runs"]) == 1
39+
assert session_in_db.memory["runs"][0]["messages"] is not None
40+
assert session_in_db.memory["runs"][0]["messages"][1]["role"] == "user"
41+
assert session_in_db.memory["runs"][0]["messages"][1]["images"] is not None
42+
assert session_in_db.memory["runs"][0]["messages"][1]["images"][0]["url"] == "https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg"
43+
44+
45+
def test_team_image_input_no_prompt(team_storage, agent_storage):
46+
image_analyst = Agent(
47+
name="Image Analyst",
48+
role="Analyze images and provide insights.",
49+
model=OpenAIChat(id="gpt-4o-mini"),
50+
markdown=True,
51+
storage=agent_storage,
52+
)
53+
54+
team = Team(
55+
model=OpenAIChat(id="gpt-4o-mini"),
56+
members=[image_analyst],
57+
name="Team",
58+
storage=team_storage,
59+
)
60+
61+
response = team.run(
62+
images=[
63+
Image(
64+
url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg"
65+
)
66+
],
67+
)
68+
assert response.content is not None
69+
70+
session_in_db = team_storage.read(response.session_id)
71+
assert session_in_db is not None
72+
assert session_in_db.memory["runs"] is not None
73+
assert len(session_in_db.memory["runs"]) == 1
74+
assert session_in_db.memory["runs"][0]["messages"] is not None
75+
assert session_in_db.memory["runs"][0]["messages"][1]["role"] == "user"
76+
assert session_in_db.memory["runs"][0]["messages"][1]["images"] is not None
77+
assert session_in_db.memory["runs"][0]["messages"][1]["images"][0]["url"] == "https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg"

0 commit comments

Comments
 (0)