Localize an element
Make one call with the Quickstart client, no history, thinking off, andtemperature=0.0. Send the screenshot (URL or base64 data URI) and an element description, and get JSON back through structured_outputs. Function calling works too: declare a click tool with the same x and y, and read tool_calls. It runs on every model except holo3-122b-a10b.
from pydantic import BaseModel, Field
MODEL_NAME = "holo4-27b"
SCREENSHOT_URL = "https://your-host/screenshot.png" # or "data:image/png;base64,..."
SCREENSHOT_WIDTH, SCREENSHOT_HEIGHT = 1280, 720
ELEMENT = "the 'Sign in' button in the top-right corner"
class VisualLocalizerOutput(BaseModel):
x: int = Field(ge=0, le=1000, description="X coordinate as integer in [0, 1000]")
y: int = Field(ge=0, le=1000, description="Y coordinate as integer in [0, 1000]")
schema = VisualLocalizerOutput.model_json_schema()
prompt = (
"Localize an element on the GUI image according to the provided target "
"and output a click position.\n"
f" * You must output a valid JSON following the format: {schema}\n"
f" Your target is:\n{ELEMENT}"
)
response = client.chat.completions.create(
model=MODEL_NAME,
messages=[{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": SCREENSHOT_URL}},
{"type": "text", "text": prompt},
],
}],
extra_body={
"structured_outputs": {"json": schema},
"chat_template_kwargs": {"enable_thinking": False},
},
temperature=0.0,
)
point = VisualLocalizerOutput.model_validate_json(response.choices[0].message.content)
abs_x = int(point.x / 1000 * SCREENSHOT_WIDTH)
abs_y = int(point.y / 1000 * SCREENSHOT_HEIGHT)
print(f"Click at ({abs_x}, {abs_y})")
const MODEL_NAME = "holo4-27b";
const SCREENSHOT_URL = "https://your-host/screenshot.png"; // or "data:image/png;base64,..."
const SCREENSHOT_WIDTH = 1280;
const SCREENSHOT_HEIGHT = 720;
const ELEMENT = "the 'Sign in' button in the top-right corner";
const schema = {
type: "object",
properties: {
x: { type: "integer", minimum: 0, maximum: 1000, description: "X coordinate as integer in [0, 1000]" },
y: { type: "integer", minimum: 0, maximum: 1000, description: "Y coordinate as integer in [0, 1000]" },
},
required: ["x", "y"],
};
const prompt =
"Localize an element on the GUI image according to the provided target " +
"and output a click position.\n" +
` * You must output a valid JSON following the format: ${JSON.stringify(schema)}\n` +
` Your target is:\n${ELEMENT}`;
const response = await client.chat.completions.create({
model: MODEL_NAME,
messages: [
{
role: "user",
content: [
{ type: "image_url", image_url: { url: SCREENSHOT_URL } },
{ type: "text", text: prompt },
],
},
],
temperature: 0.0,
// structured_outputs and chat_template_kwargs are H-specific, passed through in the request body
...({
structured_outputs: { json: schema },
chat_template_kwargs: { enable_thinking: false },
} as any),
});
const point = JSON.parse(response.choices[0].message.content!) as { x: number; y: number };
const absX = Math.round((point.x / 1000) * SCREENSHOT_WIDTH);
const absY = Math.round((point.y / 1000) * SCREENSHOT_HEIGHT);
console.log(`Click at (${absX}, ${absY})`);