feat: add look_at tool and multimodal-looker agent

Add a new tool and agent for analyzing media files (PDFs, images, diagrams)
that require visual interpretation beyond raw text.

- Add `multimodal-looker` agent using Gemini 2.5 Flash model
- Add `look_at` tool that spawns multimodal-looker sessions
- Restrict multimodal-looker from calling task/call_omo_agent/look_at tools

Inspired by Sourcegraph Ampcode's look_at tool design.

🤖 GENERATED WITH ASSISTANCE OF [OhMyOpenCode](https://github.com/code-yeongyu/oh-my-opencode)
This commit is contained in:
YeonGyu-Kim
2025-12-13 15:25:29 +09:00
parent 821b0b8e9f
commit a3938e8c25
10 changed files with 180 additions and 1 deletions

View File

@@ -0,0 +1,91 @@
import { tool, type PluginInput } from "@opencode-ai/plugin"
import { LOOK_AT_DESCRIPTION, MULTIMODAL_LOOKER_AGENT } from "./constants"
import type { LookAtArgs } from "./types"
import { log } from "../../shared/logger"
export function createLookAt(ctx: PluginInput) {
return tool({
description: LOOK_AT_DESCRIPTION,
args: {
file_path: tool.schema.string().describe("Absolute path to the file to analyze"),
goal: tool.schema.string().describe("What specific information to extract from the file"),
},
async execute(args: LookAtArgs, toolContext) {
log(`[look_at] Analyzing file: ${args.file_path}, goal: ${args.goal}`)
const prompt = `Analyze this file and extract the requested information.
File path: ${args.file_path}
Goal: ${args.goal}
Read the file using the Read tool, then provide ONLY the extracted information that matches the goal.
Be thorough on what was requested, concise on everything else.
If the requested information is not found, clearly state what is missing.`
log(`[look_at] Creating session with parent: ${toolContext.sessionID}`)
const createResult = await ctx.client.session.create({
body: {
parentID: toolContext.sessionID,
title: `look_at: ${args.goal.substring(0, 50)}`,
},
})
if (createResult.error) {
log(`[look_at] Session create error:`, createResult.error)
return `Error: Failed to create session: ${createResult.error}`
}
const sessionID = createResult.data.id
log(`[look_at] Created session: ${sessionID}`)
log(`[look_at] Sending prompt to session ${sessionID}`)
await ctx.client.session.prompt({
path: { id: sessionID },
body: {
agent: MULTIMODAL_LOOKER_AGENT,
tools: {
task: false,
call_omo_agent: false,
look_at: false,
},
parts: [{ type: "text", text: prompt }],
},
})
log(`[look_at] Prompt sent, fetching messages...`)
const messagesResult = await ctx.client.session.messages({
path: { id: sessionID },
})
if (messagesResult.error) {
log(`[look_at] Messages error:`, messagesResult.error)
return `Error: Failed to get messages: ${messagesResult.error}`
}
const messages = messagesResult.data
log(`[look_at] Got ${messages.length} messages`)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const lastAssistantMessage = messages
.filter((m: any) => m.info.role === "assistant")
.sort((a: any, b: any) => (b.info.time?.created || 0) - (a.info.time?.created || 0))[0]
if (!lastAssistantMessage) {
log(`[look_at] No assistant message found`)
return `Error: No response from multimodal-looker agent`
}
log(`[look_at] Found assistant message with ${lastAssistantMessage.parts.length} parts`)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const textParts = lastAssistantMessage.parts.filter((p: any) => p.type === "text")
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const responseText = textParts.map((p: any) => p.text).join("\n")
log(`[look_at] Got response, length: ${responseText.length}`)
return responseText
},
})
}