diff --git a/README.md b/README.md index cdebbda..7c390da 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ OpenClaw 统一私有扩展插件集合,聚合了自定义 Agent 工具、媒 | 模块类别 | 标识符 (ID) | 契约类型 | 说明 | 详细文档 | | :--- | :--- | :--- | :--- | :--- | | **视频生成** | `openai-video` | `videoGenerationProviders` | OpenAI `/v1/videos` 协议兼容视频生成驱动(支持自建网关 / gemini-web2api / Sora 协议) | [查看文档 →](src/video/README.md) | +| **图像生成** | `openai-image` | `imageGenerationProviders` | OpenAI `/v1/images` 协议兼容图像生成驱动(支持 Bifrost / ccLoad,修复 16:9 尺寸 Bug 与 Base64/URL 双模解析) | [查看文档 →](src/image/README.md) | | **基础工具** | `echo` | `tools` | 基础测试与连通性回显工具 | [查看文档 →](src/tools/README.md) | | **内置技能** | `agent-access` | `skills` | Bitwarden Agent Access 自动化技能 | [查看技能 →](skills/agent-access/SKILL.md) | @@ -57,13 +58,16 @@ openclaw-plugin-tools/ │ ├── types.d.ts # Plugin SDK 补充类型定义 │ ├── tools/ # 工具模块集合 │ │ ├── echo.ts -│ │ └── README.md # 工具专属说明 -│ └── video/ # 视频生成驱动模块 +│ │ └── README.md # 📍 工具专属说明 +│ ├── video/ # 视频生成驱动模块 +│ │ ├── index.ts +│ │ └── README.md # 📍 视频驱动专属说明 +│ └── image/ # 图像生成驱动模块 │ ├── index.ts -│ └── README.md # 视频驱动专属说明 +│ └── README.md # 📍 图像驱动专属说明 └── skills/ # 官方规范就近内置技能 └── agent-access/ - └── SKILL.md # 技能自描述文件 + └── SKILL.md # 📍 技能自描述文件 ``` --- diff --git a/dist/image/index.d.ts b/dist/image/index.d.ts new file mode 100644 index 0000000..cd51d0c --- /dev/null +++ b/dist/image/index.d.ts @@ -0,0 +1,26 @@ +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import type { ImageGenerationProvider, ImageGenerationRequest } from "openclaw/plugin-sdk/image-generation"; +export interface SecretRefObject { + source: string; + provider: string; + id: string; +} +export interface ImagePluginConfig { + baseUrl?: string; + apiKey?: string | SecretRefObject; + allowPrivateNetwork?: boolean; + defaultModel?: string; + models?: string[]; +} +export declare function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined; +export declare function resolveImageConfig(cfg: ImageGenerationRequest["cfg"] | undefined, apiPluginConfig?: Record): ImagePluginConfig | undefined; +export declare function resolveConfiguredImageBaseUrl(cfg: ImageGenerationRequest["cfg"], pluginCfg?: ImagePluginConfig): string; +export declare function resolveImageSize(params: { + size?: string; + aspectRatio?: string; + model?: string; +}): string; +export declare function buildOpenAIImageGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }: { + isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"]; + getPluginConfig?: () => Record | undefined; +}): ImageGenerationProvider; diff --git a/dist/image/index.js b/dist/image/index.js new file mode 100644 index 0000000..dc44254 --- /dev/null +++ b/dist/image/index.js @@ -0,0 +1,316 @@ +import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime"; +import { extensionForMime } from "openclaw/plugin-sdk/media-mime"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; +const DEFAULT_OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1"; +const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-2"; +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_SIZE = "1024x1024"; +const SUPPORTED_SIZES = [ + "1024x1024", + "1536x1024", + "1024x1536", + "1792x1024", + "1024x1792", + "2048x2048", + "2048x1152", + "3840x2160", +]; +const SUPPORTED_ASPECT_RATIOS = [ + "1:1", + "16:9", + "9:16", + "4:3", + "3:4", + "3:2", + "2:3", + "21:9", +]; +export function resolvePluginApiKey(rawKey) { + if (typeof rawKey === "string") { + return normalizeOptionalString(rawKey); + } + if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") { + return normalizeOptionalString(process.env[rawKey.id]); + } + return undefined; +} +export function resolveImageConfig(cfg, apiPluginConfig) { + const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config); + const legacyConfig = (cfg?.plugins?.entries?.["openai-image"]?.config ?? + cfg?.plugins?.entries?.["bifrost-image"]?.config); + const directImageConfig = toolsConfig?.image; + const flatToolsConfig = toolsConfig?.baseUrl + ? { + baseUrl: toolsConfig.baseUrl, + apiKey: toolsConfig.apiKey, + allowPrivateNetwork: toolsConfig.allowPrivateNetwork, + defaultModel: toolsConfig.defaultModel, + models: toolsConfig.models, + } + : undefined; + return { + ...legacyConfig, + ...flatToolsConfig, + ...directImageConfig, + }; +} +export function resolveConfiguredImageBaseUrl(cfg, pluginCfg) { + return (normalizeOptionalString(pluginCfg?.baseUrl) ?? + normalizeOptionalString(process.env.OPENAI_IMAGE_BASE_URL) ?? + normalizeOptionalString(process.env.BIFROST_BASE_URL) ?? + normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ?? + normalizeOptionalString(process.env.OPENAI_BASE_URL) ?? + DEFAULT_OPENAI_IMAGE_BASE_URL); +} +export function resolveImageSize(params) { + const explicitSize = normalizeOptionalString(params.size); + if (explicitSize) { + return explicitSize; + } + // Safe aspect ratio normalization preventing issue #168665 (proxy rejecting 2048x1152) + switch (normalizeOptionalString(params.aspectRatio)) { + case "1:1": + return "1024x1024"; + case "16:9": + return "1792x1024"; + case "9:16": + return "1024x1792"; + case "4:3": + case "3:2": + return "1536x1024"; + case "3:4": + case "2:3": + return "1024x1536"; + case "21:9": + return "1792x1024"; + default: + return DEFAULT_SIZE; + } +} +export function buildOpenAIImageGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }) { + return { + id: "openai-image", + aliases: ["bifrost-image", "custom-image"], + label: "OpenAI Image (Custom / Bifrost)", + defaultModel: DEFAULT_OPENAI_IMAGE_MODEL, + models: [DEFAULT_OPENAI_IMAGE_MODEL, "gpt-image-1.5", "gemini-3.1-flash-image"], + isConfigured: (ctx) => { + const pluginCfg = resolveImageConfig(ctx.cfg, getPluginConfig?.()); + if (resolvePluginApiKey(pluginCfg?.apiKey) || + Boolean(pluginCfg?.apiKey) || + normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) || + normalizeOptionalString(process.env.BIFROST_API_KEY) || + normalizeOptionalString(process.env.CCLOAD_API_KEY) || + normalizeOptionalString(process.env.OPENAI_API_KEY)) { + return true; + } + return isProviderApiKeyConfigured({ + provider: "openai", + ...ctx, + profileTypes: ["api_key"], + }); + }, + capabilities: { + generate: { + maxCount: 4, + supportsSize: true, + supportsAspectRatio: true, + supportsResolution: true, + }, + edit: { + enabled: true, + maxCount: 4, + maxInputImages: 5, + supportsSize: true, + supportsAspectRatio: true, + supportsResolution: true, + }, + geometry: { + sizes: [...SUPPORTED_SIZES], + aspectRatios: [...SUPPORTED_ASPECT_RATIOS], + resolutions: ["1K", "2K", "4K"], + }, + output: { + formats: ["png", "jpeg", "webp"], + qualities: ["low", "medium", "high", "auto"], + }, + }, + async generateImage(req) { + const pluginCfg = resolveImageConfig(req.cfg, getPluginConfig?.()); + const explicitApiKey = resolvePluginApiKey(pluginCfg?.apiKey) ?? + normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ?? + normalizeOptionalString(process.env.BIFROST_API_KEY) ?? + normalizeOptionalString(process.env.CCLOAD_API_KEY) ?? + normalizeOptionalString(process.env.OPENAI_API_KEY); + let apiKey = explicitApiKey; + if (!apiKey) { + const { resolveApiKeyForProvider } = await import("openclaw/plugin-sdk/provider-auth-runtime"); + const auth = await resolveApiKeyForProvider({ + provider: "openai", + cfg: req.cfg, + agentDir: req.agentDir, + store: req.authStore, + modelApi: "openai-responses", + }); + if (!auth.apiKey) { + throw new Error("OpenAI / Bifrost Image API key missing"); + } + apiKey = auth.apiKey; + } + const [{ assertOkOrThrowHttpError, createProviderOperationDeadline, createProviderOperationTimeoutResolver, fetchWithTimeoutGuarded, postJsonRequest, postMultipartRequest, readProviderJsonResponse, resolveProviderHttpRequestConfig, resolveProviderOperationTimeoutMs, sanitizeConfiguredModelProviderRequest, },] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]); + const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "OpenAI Image Generation", + }); + const providerConfig = req.cfg?.models?.providers?.openai; + const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true; + const requestConfig = sanitizeConfiguredModelProviderRequest({ + ...providerConfig?.request, + allowPrivateNetwork: allowPrivate, + }); + const rawBaseUrl = resolveConfiguredImageBaseUrl(req.cfg, pluginCfg); + const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ + baseUrl: rawBaseUrl, + defaultBaseUrl: DEFAULT_OPENAI_IMAGE_BASE_URL, + allowPrivateNetwork: allowPrivate, + request: requestConfig, + defaultHeaders: { + Authorization: `Bearer ${apiKey}`, + }, + provider: "openai", + capability: "image", + transport: "http", + }); + const model = normalizeOptionalString(req.model) ?? + normalizeOptionalString(pluginCfg?.defaultModel) ?? + DEFAULT_OPENAI_IMAGE_MODEL; + const size = resolveImageSize({ + size: req.size, + aspectRatio: req.aspectRatio, + model, + }); + const count = Math.max(1, Math.min(req.count ?? 1, 4)); + const inputImages = req.inputImages ?? []; + const isEdit = inputImages.length > 0; + const submitTimeoutMs = resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }); + let response; + let release; + if (isEdit) { + const form = new FormData(); + form.set("model", model); + form.set("prompt", req.prompt); + form.set("n", String(count)); + form.set("size", size); + form.set("response_format", "b64_json"); + inputImages.forEach((img, idx) => { + const mime = img.mimeType || "image/png"; + const ext = extensionForMime(mime)?.slice(1) || "png"; + const name = img.fileName || `input-${idx + 1}.${ext}`; + form.append(idx === 0 ? "image" : "image[]", new File([bufferToBlobPart(img.buffer)], name, { type: mime })); + }); + const multipartHeaders = new Headers(headers); + multipartHeaders.delete("Content-Type"); + ({ response, release } = await postMultipartRequest({ + url: `${baseUrl}/images/edits`, + headers: multipartHeaders, + body: form, + timeoutMs: submitTimeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + })); + } + else { + const jsonHeaders = new Headers(headers); + jsonHeaders.set("Content-Type", "application/json"); + const jsonBody = { + model, + prompt: req.prompt, + n: count, + size, + response_format: "b64_json", + }; + if (req.quality) { + jsonBody.quality = req.quality; + } + ({ response, release } = await postJsonRequest({ + url: `${baseUrl}/images/generations`, + headers: jsonHeaders, + body: jsonBody, + timeoutMs: submitTimeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + })); + } + let payload; + try { + await assertOkOrThrowHttpError(response, "Image generation request failed"); + payload = await readProviderJsonResponse(response, "OpenAI image generation failed"); + } + finally { + await release(); + } + if (payload.error?.message) { + throw new Error(`Image generation error: ${payload.error.message}`); + } + const dataItems = payload.data ?? []; + if (dataItems.length === 0) { + throw new Error("Image generation returned no image data"); + } + const generatedImages = []; + const downloadTimeout = createProviderOperationTimeoutResolver({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }); + for (let i = 0; i < dataItems.length; i++) { + const item = dataItems[i]; + if (item.b64_json) { + const buffer = Buffer.from(item.b64_json, "base64"); + generatedImages.push({ + buffer, + mimeType: "image/png", + fileName: `image-${i + 1}.png`, + revisedPrompt: item.revised_prompt, + }); + } + else if (item.url) { + const dlResult = await fetchWithTimeoutGuarded(item.url, { method: "GET" }, downloadTimeout(), fetchFn, { + allowPrivateNetwork, + auditContext: "openai-image-download", + }); + try { + await assertOkOrThrowHttpError(dlResult.response, "Failed to download generated image"); + const arrayBuf = await dlResult.response.arrayBuffer(); + const mime = dlResult.response.headers.get("content-type") || "image/png"; + const ext = extensionForMime(mime)?.slice(1) || "png"; + generatedImages.push({ + buffer: Buffer.from(arrayBuf), + mimeType: mime, + fileName: `image-${i + 1}.${ext}`, + revisedPrompt: item.revised_prompt, + }); + } + finally { + await dlResult.release(); + } + } + } + if (generatedImages.length === 0) { + throw new Error("Failed to extract any valid image from provider response"); + } + return { + images: generatedImages, + model, + metadata: { + created: payload.created, + size, + }, + }; + }, + }; +} diff --git a/dist/index.js b/dist/index.js index 6da5f92..2d72a77 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3,11 +3,13 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin"; import { echoTool } from "./tools/echo.js"; import { buildOpenAIVideoGenerationProvider } from "./video/index.js"; +import { buildOpenAIImageGenerationProvider } from "./image/index.js"; const entry = definePluginEntry({ id: "tools", name: "Tools", description: "Custom Tools and Media Providers for OpenClaw.", register(api) { + // 1. Register tools api.registerTool({ name: echoTool.name, label: echoTool.label, @@ -18,10 +20,16 @@ const entry = definePluginEntry({ return jsonResult(result); }, }); + // 2. Register video generation provider api.registerVideoGenerationProvider(buildOpenAIVideoGenerationProvider({ isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured, getPluginConfig: () => api.pluginConfig, })); + // 3. Register image generation provider + api.registerImageGenerationProvider(buildOpenAIImageGenerationProvider({ + isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured, + getPluginConfig: () => api.pluginConfig, + })); }, }); Object.defineProperty(entry, toolPluginMetadataSymbol, { diff --git a/openclaw.plugin.json b/openclaw.plugin.json index a2b7b7f..c237aeb 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -1,8 +1,8 @@ { "id": "tools", "name": "Tools", - "description": "Custom Tools and Media Providers for OpenClaw.", - "version": "0.2.0", + "description": "Custom Tools and Media Providers (Video & Image) for OpenClaw.", + "version": "0.3.0", "categories": [ "media", "utilities" @@ -23,6 +23,15 @@ "BIFROST_API_KEY", "OPENAI_API_KEY" ] + }, + { + "id": "openai-image", + "envVars": [ + "OPENAI_IMAGE_API_KEY", + "BIFROST_API_KEY", + "CCLOAD_API_KEY", + "OPENAI_API_KEY" + ] } ] }, @@ -32,6 +41,9 @@ ], "videoGenerationProviders": [ "openai-video" + ], + "imageGenerationProviders": [ + "openai-image" ] }, "videoGenerationProviderMetadata": { @@ -43,6 +55,15 @@ ] } }, + "imageGenerationProviderMetadata": { + "openai-image": { + "authSignals": [ + { + "provider": "openai" + } + ] + } + }, "configContracts": { "secretInputs": { "paths": [ @@ -55,6 +76,11 @@ "path": "video.apiKey", "expected": "string", "ownerKind": "capability" + }, + { + "path": "image.apiKey", + "expected": "string", + "ownerKind": "capability" } ] } @@ -67,6 +93,10 @@ "video.apiKey": { "label": "Video API Key", "sensitive": true + }, + "image.apiKey": { + "label": "Image API Key", + "sensitive": true } }, "configSchema": { @@ -76,7 +106,7 @@ "baseUrl": { "type": "string", "title": "API Base URL", - "description": "Base URL of the OpenAI-compatible video endpoint" + "description": "Base URL of the OpenAI-compatible endpoint" }, "apiKey": { "type": [ @@ -84,7 +114,7 @@ "object" ], "title": "API Key", - "description": "API Key or SecretRef for video generation" + "description": "API Key or SecretRef" }, "allowPrivateNetwork": { "type": "boolean", @@ -94,27 +124,20 @@ }, "defaultModel": { "type": "string", - "title": "Default Video Model", - "description": "Default model ID to use when no model is explicitly requested", - "default": "sora-2" + "title": "Default Model", + "description": "Default model ID" }, "models": { "type": "array", "title": "Available Models", - "description": "List of available video model IDs", "items": { "type": "string" - }, - "default": [ - "sora-2", - "sora-2-pro", - "gemini-video" - ] + } }, "video": { "type": "object", "title": "Video Generation Settings", - "description": "Nested configuration for video generation", + "description": "Configuration for video generation (openai-video)", "additionalProperties": false, "properties": { "baseUrl": { @@ -141,6 +164,37 @@ } } } + }, + "image": { + "type": "object", + "title": "Image Generation Settings", + "description": "Configuration for image generation (openai-image / bifrost-image)", + "additionalProperties": false, + "properties": { + "baseUrl": { + "type": "string" + }, + "apiKey": { + "type": [ + "string", + "object" + ] + }, + "allowPrivateNetwork": { + "type": "boolean", + "default": true + }, + "defaultModel": { + "type": "string", + "default": "gpt-image-2" + }, + "models": { + "type": "array", + "items": { + "type": "string" + } + } + } } } } diff --git a/src/image/README.md b/src/image/README.md new file mode 100644 index 0000000..10a8830 --- /dev/null +++ b/src/image/README.md @@ -0,0 +1,76 @@ +# OpenAI Image Provider (`openai-image` / `bifrost-image`) + +OpenClaw 图像生成驱动模块,适配标准 **OpenAI Images API**(`/v1/images/generations` 及 `/v1/images/edits` 规范)。 + +专为自建聚合网关或第三方中转(如 `Bifrost`、`ccLoad`)定制打造: +* **尺寸安全归一化**:修复官方驱动在反代环境下因 `16:9` 映射为 `2048x1152` 导致上游报 400 错误的 Bug(Issue #168665),自动将 `16:9` / `9:16` 适配为上游通用的安全比例规格(如 `1792x1024` / `1536x1024`)。 +* **格式双模自适应**:同时支持解析上游返回的 `b64_json`(Base64 内存直解码)与 `url`(自动安全下载并保存为本地 Buffer),彻底杜绝 `response malformed` 解析失败。 +* **图生图编辑支持**:当传入 `inputImages` 时,自动走 Multipart 表单发起编辑请求;纯文本生图时走轻量 JSON 请求。 +* **私有网络与反代直通**:原生支持私有/内网端点请求(`allowPrivateNetwork: true`),兼容 `127.0.0.1`、`host.incus.internal` 或自建反代网关。 + +--- + +## 标识符与别名 + +* **Provider ID**:`openai-image` +* **支持别名 (Aliases)**:`bifrost-image`, `custom-image` +* **默认模型**:`gpt-image-2` +* **支持模型列表**:`gpt-image-2`, `gpt-image-1.5`, `gemini-3.1-flash-image`(也可在配置中自定义扩展) + +--- + +## 配置说明 + +支持通过 **Web 控制台(Control UI)** 或直接编辑 `~/.openclaw/openclaw.json`。 + +### 方式 1:Web 控制台可视化配置 +进入 **Settings → Plugins → Tools**: +* **Image Base URL**:填写图像上游端点(例如 `https://Direct-Bifrost.oo1.dev/v1` 或内部反代地址) +* **Image API Key**:填写 API Key 或 SecretRef(留空时自动回退读取环境变量 `OPENAI_IMAGE_API_KEY`、`BIFROST_API_KEY`、`CCLOAD_API_KEY` 或 `OPENAI_API_KEY`) +* **Default Image Model**:例如 `gpt-image-2` + +### 方式 2:编辑 `openclaw.json` + +在 `~/.openclaw/openclaw.json` 中配置: + +```json5 +{ + "plugins": { + "entries": { + "tools": { + "enabled": true, + "config": { + "image": { + "baseUrl": "https://Direct-Bifrost.oo1.dev/v1", + "apiKey": "sk-your-bifrost-key", + "defaultModel": "gpt-image-2", + "allowPrivateNetwork": true + } + } + } + } + }, + "agents": { + "defaults": { + "mediaModels": { + "image": { + "primary": "openai-image/gpt-image-2", + "fallbacks": [ + "openai-image/gemini-3.1-flash-image" + ] + } + } + } + } +} +``` + +--- + +## 使用方法 + +配置完成后,OpenClaw 内置的 `image_generate` 工具即可自动调度: +* **文本生成图像**: + > “生成一张 16:9 的秋日晨光现代书房桌面壁纸” +* **垫图修改/编辑**: + 在对话中附带图片,提示助手基于该图进行风格调整或重绘。 diff --git a/src/image/index.ts b/src/image/index.ts new file mode 100644 index 0000000..f5edfa6 --- /dev/null +++ b/src/image/index.ts @@ -0,0 +1,444 @@ +import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime"; +import { extensionForMime } from "openclaw/plugin-sdk/media-mime"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; +import type { + GeneratedImageAsset, + ImageGenerationProvider, + ImageGenerationRequest, + ImageGenerationResult, +} from "openclaw/plugin-sdk/image-generation"; + +const DEFAULT_OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1"; +const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-2"; +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_SIZE = "1024x1024"; + +const SUPPORTED_SIZES = [ + "1024x1024", + "1536x1024", + "1024x1536", + "1792x1024", + "1024x1792", + "2048x2048", + "2048x1152", + "3840x2160", +] as const; + +const SUPPORTED_ASPECT_RATIOS = [ + "1:1", + "16:9", + "9:16", + "4:3", + "3:4", + "3:2", + "2:3", + "21:9", +] as const; + +export interface SecretRefObject { + source: string; + provider: string; + id: string; +} + +export interface ImagePluginConfig { + baseUrl?: string; + apiKey?: string | SecretRefObject; + allowPrivateNetwork?: boolean; + defaultModel?: string; + models?: string[]; +} + +export function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined { + if (typeof rawKey === "string") { + return normalizeOptionalString(rawKey); + } + if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") { + return normalizeOptionalString(process.env[rawKey.id]); + } + return undefined; +} + +export function resolveImageConfig( + cfg: ImageGenerationRequest["cfg"] | undefined, + apiPluginConfig?: Record, +): ImagePluginConfig | undefined { + const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config) as + | { + image?: ImagePluginConfig; + baseUrl?: string; + apiKey?: string | SecretRefObject; + defaultModel?: string; + models?: string[]; + allowPrivateNetwork?: boolean; + } + | undefined; + + const legacyConfig = ( + cfg?.plugins?.entries?.["openai-image"]?.config ?? + cfg?.plugins?.entries?.["bifrost-image"]?.config + ) as ImagePluginConfig | undefined; + + const directImageConfig = toolsConfig?.image; + const flatToolsConfig: ImagePluginConfig | undefined = toolsConfig?.baseUrl + ? { + baseUrl: toolsConfig.baseUrl, + apiKey: toolsConfig.apiKey, + allowPrivateNetwork: toolsConfig.allowPrivateNetwork, + defaultModel: toolsConfig.defaultModel, + models: toolsConfig.models, + } + : undefined; + + return { + ...legacyConfig, + ...flatToolsConfig, + ...directImageConfig, + }; +} + +export function resolveConfiguredImageBaseUrl( + cfg: ImageGenerationRequest["cfg"], + pluginCfg?: ImagePluginConfig, +): string { + return ( + normalizeOptionalString(pluginCfg?.baseUrl) ?? + normalizeOptionalString(process.env.OPENAI_IMAGE_BASE_URL) ?? + normalizeOptionalString(process.env.BIFROST_BASE_URL) ?? + normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ?? + normalizeOptionalString(process.env.OPENAI_BASE_URL) ?? + DEFAULT_OPENAI_IMAGE_BASE_URL + ); +} + +export function resolveImageSize(params: { + size?: string; + aspectRatio?: string; + model?: string; +}): string { + const explicitSize = normalizeOptionalString(params.size); + if (explicitSize) { + return explicitSize; + } + + // Safe aspect ratio normalization preventing issue #168665 (proxy rejecting 2048x1152) + switch (normalizeOptionalString(params.aspectRatio)) { + case "1:1": + return "1024x1024"; + case "16:9": + return "1792x1024"; + case "9:16": + return "1024x1792"; + case "4:3": + case "3:2": + return "1536x1024"; + case "3:4": + case "2:3": + return "1024x1536"; + case "21:9": + return "1792x1024"; + default: + return DEFAULT_SIZE; + } +} + +export function buildOpenAIImageGenerationProvider({ + isProviderApiKeyConfigured, + getPluginConfig, +}: { + isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"]; + getPluginConfig?: () => Record | undefined; +}): ImageGenerationProvider { + return { + id: "openai-image", + aliases: ["bifrost-image", "custom-image"], + label: "OpenAI Image (Custom / Bifrost)", + defaultModel: DEFAULT_OPENAI_IMAGE_MODEL, + models: [DEFAULT_OPENAI_IMAGE_MODEL, "gpt-image-1.5", "gemini-3.1-flash-image"], + isConfigured: (ctx) => { + const pluginCfg = resolveImageConfig(ctx.cfg, getPluginConfig?.()); + if ( + resolvePluginApiKey(pluginCfg?.apiKey) || + Boolean(pluginCfg?.apiKey) || + normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) || + normalizeOptionalString(process.env.BIFROST_API_KEY) || + normalizeOptionalString(process.env.CCLOAD_API_KEY) || + normalizeOptionalString(process.env.OPENAI_API_KEY) + ) { + return true; + } + return isProviderApiKeyConfigured({ + provider: "openai", + ...ctx, + profileTypes: ["api_key"], + }); + }, + capabilities: { + generate: { + maxCount: 4, + supportsSize: true, + supportsAspectRatio: true, + supportsResolution: true, + }, + edit: { + enabled: true, + maxCount: 4, + maxInputImages: 5, + supportsSize: true, + supportsAspectRatio: true, + supportsResolution: true, + }, + geometry: { + sizes: [...SUPPORTED_SIZES], + aspectRatios: [...SUPPORTED_ASPECT_RATIOS], + resolutions: ["1K", "2K", "4K"], + }, + output: { + formats: ["png", "jpeg", "webp"], + qualities: ["low", "medium", "high", "auto"], + }, + }, + async generateImage(req: ImageGenerationRequest): Promise { + const pluginCfg = resolveImageConfig(req.cfg, getPluginConfig?.()); + const explicitApiKey = + resolvePluginApiKey(pluginCfg?.apiKey) ?? + normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ?? + normalizeOptionalString(process.env.BIFROST_API_KEY) ?? + normalizeOptionalString(process.env.CCLOAD_API_KEY) ?? + normalizeOptionalString(process.env.OPENAI_API_KEY); + + let apiKey = explicitApiKey; + if (!apiKey) { + const { resolveApiKeyForProvider } = await import( + "openclaw/plugin-sdk/provider-auth-runtime" + ); + const auth = await resolveApiKeyForProvider({ + provider: "openai", + cfg: req.cfg, + agentDir: req.agentDir, + store: req.authStore, + modelApi: "openai-responses", + }); + if (!auth.apiKey) { + throw new Error("OpenAI / Bifrost Image API key missing"); + } + apiKey = auth.apiKey; + } + + const [ + { + assertOkOrThrowHttpError, + createProviderOperationDeadline, + createProviderOperationTimeoutResolver, + fetchWithTimeoutGuarded, + postJsonRequest, + postMultipartRequest, + readProviderJsonResponse, + resolveProviderHttpRequestConfig, + resolveProviderOperationTimeoutMs, + sanitizeConfiguredModelProviderRequest, + }, + ] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]); + + const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "OpenAI Image Generation", + }); + + const providerConfig = req.cfg?.models?.providers?.openai; + const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true; + const requestConfig = sanitizeConfiguredModelProviderRequest({ + ...providerConfig?.request, + allowPrivateNetwork: allowPrivate, + }); + + const rawBaseUrl = resolveConfiguredImageBaseUrl(req.cfg, pluginCfg); + const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = + resolveProviderHttpRequestConfig({ + baseUrl: rawBaseUrl, + defaultBaseUrl: DEFAULT_OPENAI_IMAGE_BASE_URL, + allowPrivateNetwork: allowPrivate, + request: requestConfig, + defaultHeaders: { + Authorization: `Bearer ${apiKey}`, + }, + provider: "openai", + capability: "image", + transport: "http", + }); + + const model = + normalizeOptionalString(req.model) ?? + normalizeOptionalString(pluginCfg?.defaultModel) ?? + DEFAULT_OPENAI_IMAGE_MODEL; + + const size = resolveImageSize({ + size: req.size, + aspectRatio: req.aspectRatio, + model, + }); + + const count = Math.max(1, Math.min(req.count ?? 1, 4)); + const inputImages = req.inputImages ?? []; + const isEdit = inputImages.length > 0; + + const submitTimeoutMs = resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }); + + let response: Response; + let release: () => Promise; + + if (isEdit) { + const form = new FormData(); + form.set("model", model); + form.set("prompt", req.prompt); + form.set("n", String(count)); + form.set("size", size); + form.set("response_format", "b64_json"); + + inputImages.forEach((img, idx) => { + const mime = img.mimeType || "image/png"; + const ext = extensionForMime(mime)?.slice(1) || "png"; + const name = img.fileName || `input-${idx + 1}.${ext}`; + form.append( + idx === 0 ? "image" : "image[]", + new File([bufferToBlobPart(img.buffer)], name, { type: mime }), + ); + }); + + const multipartHeaders = new Headers(headers); + multipartHeaders.delete("Content-Type"); + + ({ response, release } = await postMultipartRequest({ + url: `${baseUrl}/images/edits`, + headers: multipartHeaders, + body: form, + timeoutMs: submitTimeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + })); + } else { + const jsonHeaders = new Headers(headers); + jsonHeaders.set("Content-Type", "application/json"); + + const jsonBody: Record = { + model, + prompt: req.prompt, + n: count, + size, + response_format: "b64_json", + }; + + if (req.quality) { + jsonBody.quality = req.quality; + } + + ({ response, release } = await postJsonRequest({ + url: `${baseUrl}/images/generations`, + headers: jsonHeaders, + body: jsonBody, + timeoutMs: submitTimeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + })); + } + + interface OpenAIImageApiResponse { + created?: number; + data?: Array<{ + b64_json?: string; + url?: string; + revised_prompt?: string; + }>; + error?: { + message?: string; + code?: string; + type?: string; + }; + } + + let payload: OpenAIImageApiResponse; + try { + await assertOkOrThrowHttpError(response, "Image generation request failed"); + payload = await readProviderJsonResponse( + response, + "OpenAI image generation failed", + ); + } finally { + await release(); + } + + if (payload.error?.message) { + throw new Error(`Image generation error: ${payload.error.message}`); + } + + const dataItems = payload.data ?? []; + if (dataItems.length === 0) { + throw new Error("Image generation returned no image data"); + } + + const generatedImages: GeneratedImageAsset[] = []; + const downloadTimeout = createProviderOperationTimeoutResolver({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }); + + for (let i = 0; i < dataItems.length; i++) { + const item = dataItems[i]; + if (item.b64_json) { + const buffer = Buffer.from(item.b64_json, "base64"); + generatedImages.push({ + buffer, + mimeType: "image/png", + fileName: `image-${i + 1}.png`, + revisedPrompt: item.revised_prompt, + }); + } else if (item.url) { + const dlResult = await fetchWithTimeoutGuarded( + item.url, + { method: "GET" }, + downloadTimeout(), + fetchFn, + { + allowPrivateNetwork, + auditContext: "openai-image-download", + }, + ); + try { + await assertOkOrThrowHttpError(dlResult.response, "Failed to download generated image"); + const arrayBuf = await dlResult.response.arrayBuffer(); + const mime = dlResult.response.headers.get("content-type") || "image/png"; + const ext = extensionForMime(mime)?.slice(1) || "png"; + generatedImages.push({ + buffer: Buffer.from(arrayBuf), + mimeType: mime, + fileName: `image-${i + 1}.${ext}`, + revisedPrompt: item.revised_prompt, + }); + } finally { + await dlResult.release(); + } + } + } + + if (generatedImages.length === 0) { + throw new Error("Failed to extract any valid image from provider response"); + } + + return { + images: generatedImages, + model, + metadata: { + created: payload.created, + size, + }, + }; + }, + }; +} diff --git a/src/index.test.ts b/src/index.test.ts index 9ed153d..7fa98c5 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -7,9 +7,10 @@ describe("tools", () => { expect(getToolPluginMetadata(entry)?.tools.map((tool) => tool.name)).toEqual(["echo"]); }); - it("registers both tool and video generation provider", () => { + it("registers tool, video generation provider, and image generation provider", () => { const registeredTools: unknown[] = []; const registeredVideoProviders: unknown[] = []; + const registeredImageProviders: unknown[] = []; const mockApi = { runtime: { @@ -22,6 +23,9 @@ describe("tools", () => { registerVideoGenerationProvider: vi.fn((provider) => registeredVideoProviders.push(provider), ), + registerImageGenerationProvider: vi.fn((provider) => + registeredImageProviders.push(provider), + ), }; entry.register(mockApi as any); @@ -31,5 +35,8 @@ describe("tools", () => { expect(mockApi.registerVideoGenerationProvider).toHaveBeenCalledTimes(1); expect(registeredVideoProviders[0]).toMatchObject({ id: "openai-video" }); + + expect(mockApi.registerImageGenerationProvider).toHaveBeenCalledTimes(1); + expect(registeredImageProviders[0]).toMatchObject({ id: "openai-image" }); }); }); diff --git a/src/index.ts b/src/index.ts index 62089d6..15a6d2d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,12 +3,14 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin"; import { echoTool } from "./tools/echo.js"; import { buildOpenAIVideoGenerationProvider } from "./video/index.js"; +import { buildOpenAIImageGenerationProvider } from "./image/index.js"; const entry = definePluginEntry({ id: "tools", name: "Tools", description: "Custom Tools and Media Providers for OpenClaw.", register(api) { + // 1. Register tools api.registerTool({ name: echoTool.name, label: echoTool.label, @@ -20,12 +22,21 @@ const entry = definePluginEntry({ }, }); + // 2. Register video generation provider api.registerVideoGenerationProvider( buildOpenAIVideoGenerationProvider({ isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured, getPluginConfig: () => api.pluginConfig, }), ); + + // 3. Register image generation provider + api.registerImageGenerationProvider( + buildOpenAIImageGenerationProvider({ + isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured, + getPluginConfig: () => api.pluginConfig, + }), + ); }, }); diff --git a/src/types.d.ts b/src/types.d.ts index 33f8a63..8ca192d 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -30,6 +30,58 @@ declare module "openclaw/plugin-sdk/video-generation" { } } +declare module "openclaw/plugin-sdk/image-generation" { + export interface GeneratedImageAsset { + buffer: Buffer; + mimeType: string; + fileName?: string; + revisedPrompt?: string; + metadata?: Record; + } + + export interface ImageGenerationRequest { + prompt: string; + model?: string; + count?: number; + size?: string; + aspectRatio?: string; + resolution?: string; + quality?: string; + outputFormat?: string; + background?: string; + inputImages?: Array<{ buffer: Buffer; mimeType?: string; fileName?: string }>; + cfg?: any; + agentDir?: string; + authStore?: any; + timeoutMs?: number; + ssrfPolicy?: any; + providerOptions?: Record; + } + + export interface ImageGenerationResult { + images: GeneratedImageAsset[]; + model?: string; + metadata?: Record; + } + + export interface ImageGenerationProvider { + id: string; + aliases?: string[]; + label?: string; + defaultModel?: string; + defaultTimeoutMs?: number; + models?: string[]; + capabilities: any; + isConfigured?: (ctx: { cfg?: any }) => boolean; + generateImage: (req: ImageGenerationRequest) => Promise; + } + + export function parseOpenAiCompatibleImageResponse( + payload: unknown, + options?: { defaultMimeType?: string; malformedResponseError?: string; fileNamePrefix?: string; sniffMimeType?: boolean }, + ): GeneratedImageAsset[]; +} + declare module "openclaw/plugin-sdk/provider-http" { export function createProviderOperationDeadline(params: { timeoutMs?: number; label: string }): any; export function createProviderOperationTimeoutResolver(params: { deadline: any; defaultTimeoutMs: number }): () => number; @@ -39,7 +91,7 @@ declare module "openclaw/plugin-sdk/provider-http" { export function assertOkOrThrowHttpError(response: Response, message: string): Promise; export function postMultipartRequest(params: any): Promise<{ response: Response; release: () => Promise }>; export function postJsonRequest(params: any): Promise<{ response: Response; release: () => Promise }>; - export function readProviderJsonResponse(response: Response, message: string): Promise; + export function readProviderJsonResponse(response: Response, message: string, opts?: any): Promise; export function pollProviderOperationJson(params: any): Promise; export function resolveProviderOperationTimeoutMs(params: any): number; export function resolveProviderHttpRequestConfig(params: any): any;