Multimodal agents — vision, image, and audio in one pipeline

from agentfield import Agent, AIConfig
from agentfield.multimodal import image_from_url, image_from_file, audio_from_file, text

app = Agent(
    node_id="creative-agent",
    ai_config=AIConfig(model="openai/gpt-4o"),
)

@app.reasoner()
async def storyboard_from_brief(brief: str, reference_url: str) -> dict:
    # Vision input — auto-detects the image URL
    interpretation = await app.ai(
        text("Describe the visual style of this reference:"),
        image_from_url(reference_url),
    )

    # Generate three images matching the brief
    images = await app.ai_generate_image(
        f"{brief}. Style: {interpretation}",
        model="dall-e-3",
        size="1792x1024",
        num_images=3,
    )
    saved = images.save_all("./output", prefix="frame")

    # Voice-over from the brief
    audio = await app.ai_generate_audio(
        f"Storyboard concept: {brief}",
        model="tts-1-hd",
        voice="alloy",
    )
    audio.audio.save("./output/voiceover.wav")

    return {
        "interpretation": str(interpretation),
        "frames": list(saved.values()),
        "voiceover": "./output/voiceover.wav",
    }

# Audio input — transcribe and act on a recording
@app.reasoner()
async def summarize_meeting(recording_path: str) -> dict:
    return await app.ai(
        text("Transcribe this meeting and pull out the action items:"),
        audio_from_file(recording_path),
    )

app.run()

import { Agent, OpenRouterMediaProvider } from "@agentfield/sdk";
import { openai } from "@ai-sdk/openai";
import { generateText } from "ai";

const app = new Agent({
  nodeId: "creative-agent",
  aiConfig: { provider: "openai", model: "gpt-4o" },
});

const media = new OpenRouterMediaProvider();

app.reasoner("storyboard_from_brief", async (ctx) => {
  const { brief, referenceUrl } = ctx.input;

  // Vision input — pass the image as a content part via the Vercel AI SDK
  const { text: interpretation } = await generateText({
    model: openai("gpt-4o"),
    messages: [
      {
        role: "user",
        content: [
          { type: "text", text: "Describe the visual style of this reference:" },
          { type: "image", image: new URL(referenceUrl) },
        ],
      },
    ],
  });

  // Generate an image matching the brief
  const images = await media.generateImage({
    prompt: `${brief}. Style: ${interpretation}`,
    model: "openrouter/google/gemini-3.1-flash-image-preview",
    size: "1792x1024",
  });

  // Voice-over from the brief
  const audio = await media.generateAudio({
    text: `Storyboard concept: ${brief}`,
    model: "openrouter/openai/tts-1",
    voice: "alloy",
  });

  return {
    interpretation,
    frame: images.images[0]?.url ?? images.images[0]?.b64Json,
    voiceover: audio.audio?.url ?? audio.audio?.data,
  };
});

app.serve();

package main

import (
    "context"
    "fmt"
    "log"

    "github.com/Agent-Field/agentfield/sdk/go/agent"
    "github.com/Agent-Field/agentfield/sdk/go/ai"
)

func main() {
    a, err := agent.New(agent.Config{
        NodeID:        "creative-agent",
        Version:       "1.0.0",
        AgentFieldURL: "http://localhost:8080",
        AIConfig:      &ai.Config{Model: "openai/gpt-4o"},
    })
    if err != nil {
        log.Fatal(err)
    }

    media, err := ai.NewOpenRouterMediaProvider("") // reads OPENROUTER_API_KEY
    if err != nil {
        log.Fatal(err)
    }

    a.RegisterReasoner("storyboard_from_brief", func(ctx context.Context, input map[string]any) (any, error) {
        brief := fmt.Sprintf("%v", input["brief"])
        referenceURL := fmt.Sprintf("%v", input["reference_url"])

        // Vision input — attach the reference image to the request
        resp, err := a.AI(ctx, "Describe the visual style of this reference:",
            ai.WithImageURL(referenceURL),
        )
        if err != nil {
            return nil, err
        }
        interpretation := resp.Text()

        // Generate an image matching the brief
        images, err := media.GenerateImage(ctx, ai.ImageRequest{
            Prompt: fmt.Sprintf("%s. Style: %s", brief, interpretation),
            Model:  "openrouter/google/gemini-3.1-flash-image-preview",
            Size:   "1792x1024",
        })
        if err != nil {
            return nil, err
        }

        // Voice-over from the brief
        audio, err := media.GenerateAudio(ctx, ai.AudioRequest{
            Text:  "Storyboard concept: " + brief,
            Model: "openrouter/openai/tts-1",
            Voice: "alloy",
        })
        if err != nil {
            return nil, err
        }

        return map[string]any{
            "interpretation": interpretation,
            "frame":          images.Images[0].URL,
            "voiceover":      audio.Audio.URL,
        }, nil
    })

    a.Serve(context.Background())
}