feat(image): add OpenAI/Bifrost compatible image generation provider

This commit is contained in:
openclaw committed 2026-10-19 02:16:01 +00:00
1 parent 93ae5b7342
commit 2011077fcd
10 files changed
+1019 -21

No files matched your search

+76
View File
@@ -0,0 +1,76 @@
# OpenAI Image Provider (`openai-image` / `bifrost-image`)
OpenClaw 图像生成驱动模块,适配标准 **OpenAI Images API**(`/v1/images/generations` 及 `/v1/images/edits` 规范)。
专为自建聚合网关或第三方中转(如 `Bifrost`、`ccLoad`)定制打造:
* **尺寸安全归一化**:修复官方驱动在反代环境下因 `16:9` 映射为 `2048x1152` 导致上游报 400 错误的 Bug(Issue #168665),自动将 `16:9` / `9:16` 适配为上游通用的安全比例规格(如 `1792x1024` / `1536x1024`)。
* **格式双模自适应**:同时支持解析上游返回的 `b64_json`(Base64 内存直解码)与 `url`(自动安全下载并保存为本地 Buffer),彻底杜绝 `response malformed` 解析失败。
* **图生图编辑支持**:当传入 `inputImages` 时,自动走 Multipart 表单发起编辑请求;纯文本生图时走轻量 JSON 请求。
* **私有网络与反代直通**:原生支持私有/内网端点请求(`allowPrivateNetwork: true`),兼容 `127.0.0.1`、`host.incus.internal` 或自建反代网关。
---
## 标识符与别名
* **Provider ID**:`openai-image`
* **支持别名 (Aliases)**:`bifrost-image`, `custom-image`
* **默认模型**:`gpt-image-2`
* **支持模型列表**:`gpt-image-2`, `gpt-image-1.5`, `gemini-3.1-flash-image`(也可在配置中自定义扩展)
---
## 配置说明
支持通过 **Web 控制台(Control UI)** 或直接编辑 `~/.openclaw/openclaw.json`。
### 方式 1:Web 控制台可视化配置
进入 **Settings → Plugins → Tools**:
* **Image Base URL**:填写图像上游端点(例如 `https://Direct-Bifrost.oo1.dev/v1` 或内部反代地址)
* **Image API Key**:填写 API Key 或 SecretRef(留空时自动回退读取环境变量 `OPENAI_IMAGE_API_KEY`、`BIFROST_API_KEY`、`CCLOAD_API_KEY` 或 `OPENAI_API_KEY`)
* **Default Image Model**:例如 `gpt-image-2`
### 方式 2:编辑 `openclaw.json`
在 `~/.openclaw/openclaw.json` 中配置:
```json5
{
"plugins": {
"entries": {
"tools": {
"enabled": true,
"config": {
"image": {
"baseUrl": "https://Direct-Bifrost.oo1.dev/v1",
"apiKey": "sk-your-bifrost-key",
"defaultModel": "gpt-image-2",
"allowPrivateNetwork": true
}
}
}
}
},
"agents": {
"defaults": {
"mediaModels": {
"image": {
"primary": "openai-image/gpt-image-2",
"fallbacks": [
"openai-image/gemini-3.1-flash-image"
]
}
}
}
}
}
```
---
## 使用方法
配置完成后,OpenClaw 内置的 `image_generate` 工具即可自动调度:
* **文本生成图像**:
> “生成一张 16:9 的秋日晨光现代书房桌面壁纸”
* **垫图修改/编辑**:
在对话中附带图片,提示助手基于该图进行风格调整或重绘。
+444
View File
@@ -0,0 +1,444 @@
import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime";
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
import type {
GeneratedImageAsset,
ImageGenerationProvider,
ImageGenerationRequest,
ImageGenerationResult,
} from "openclaw/plugin-sdk/image-generation";
const DEFAULT_OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-2";
const DEFAULT_TIMEOUT_MS = 120_000;
const DEFAULT_SIZE = "1024x1024";
const SUPPORTED_SIZES = [
"1024x1024",
"1536x1024",
"1024x1536",
"1792x1024",
"1024x1792",
"2048x2048",
"2048x1152",
"3840x2160",
] as const;
const SUPPORTED_ASPECT_RATIOS = [
"1:1",
"16:9",
"9:16",
"4:3",
"3:4",
"3:2",
"2:3",
"21:9",
] as const;
export interface SecretRefObject {
source: string;
provider: string;
id: string;
}
export interface ImagePluginConfig {
baseUrl?: string;
apiKey?: string | SecretRefObject;
allowPrivateNetwork?: boolean;
defaultModel?: string;
models?: string[];
}
export function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined {
if (typeof rawKey === "string") {
return normalizeOptionalString(rawKey);
}
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
return normalizeOptionalString(process.env[rawKey.id]);
}
return undefined;
}
export function resolveImageConfig(
cfg: ImageGenerationRequest["cfg"] | undefined,
apiPluginConfig?: Record<string, unknown>,
): ImagePluginConfig | undefined {
const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config) as
| {
image?: ImagePluginConfig;
baseUrl?: string;
apiKey?: string | SecretRefObject;
defaultModel?: string;
models?: string[];
allowPrivateNetwork?: boolean;
}
| undefined;
const legacyConfig = (
cfg?.plugins?.entries?.["openai-image"]?.config ??
cfg?.plugins?.entries?.["bifrost-image"]?.config
) as ImagePluginConfig | undefined;
const directImageConfig = toolsConfig?.image;
const flatToolsConfig: ImagePluginConfig | undefined = toolsConfig?.baseUrl
? {
baseUrl: toolsConfig.baseUrl,
apiKey: toolsConfig.apiKey,
allowPrivateNetwork: toolsConfig.allowPrivateNetwork,
defaultModel: toolsConfig.defaultModel,
models: toolsConfig.models,
}
: undefined;
return {
...legacyConfig,
...flatToolsConfig,
...directImageConfig,
};
}
export function resolveConfiguredImageBaseUrl(
cfg: ImageGenerationRequest["cfg"],
pluginCfg?: ImagePluginConfig,
): string {
return (
normalizeOptionalString(pluginCfg?.baseUrl) ??
normalizeOptionalString(process.env.OPENAI_IMAGE_BASE_URL) ??
normalizeOptionalString(process.env.BIFROST_BASE_URL) ??
normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ??
normalizeOptionalString(process.env.OPENAI_BASE_URL) ??
DEFAULT_OPENAI_IMAGE_BASE_URL
);
}
export function resolveImageSize(params: {
size?: string;
aspectRatio?: string;
model?: string;
}): string {
const explicitSize = normalizeOptionalString(params.size);
if (explicitSize) {
return explicitSize;
}
// Safe aspect ratio normalization preventing issue #168665 (proxy rejecting 2048x1152)
switch (normalizeOptionalString(params.aspectRatio)) {
case "1:1":
return "1024x1024";
case "16:9":
return "1792x1024";
case "9:16":
return "1024x1792";
case "4:3":
case "3:2":
return "1536x1024";
case "3:4":
case "2:3":
return "1024x1536";
case "21:9":
return "1792x1024";
default:
return DEFAULT_SIZE;
}
}
export function buildOpenAIImageGenerationProvider({
isProviderApiKeyConfigured,
getPluginConfig,
}: {
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
getPluginConfig?: () => Record<string, unknown> | undefined;
}): ImageGenerationProvider {
return {
id: "openai-image",
aliases: ["bifrost-image", "custom-image"],
label: "OpenAI Image (Custom / Bifrost)",
defaultModel: DEFAULT_OPENAI_IMAGE_MODEL,
models: [DEFAULT_OPENAI_IMAGE_MODEL, "gpt-image-1.5", "gemini-3.1-flash-image"],
isConfigured: (ctx) => {
const pluginCfg = resolveImageConfig(ctx.cfg, getPluginConfig?.());
if (
resolvePluginApiKey(pluginCfg?.apiKey) ||
Boolean(pluginCfg?.apiKey) ||
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ||
normalizeOptionalString(process.env.BIFROST_API_KEY) ||
normalizeOptionalString(process.env.CCLOAD_API_KEY) ||
normalizeOptionalString(process.env.OPENAI_API_KEY)
) {
return true;
}
return isProviderApiKeyConfigured({
provider: "openai",
...ctx,
profileTypes: ["api_key"],
});
},
capabilities: {
generate: {
maxCount: 4,
supportsSize: true,
supportsAspectRatio: true,
supportsResolution: true,
},
edit: {
enabled: true,
maxCount: 4,
maxInputImages: 5,
supportsSize: true,
supportsAspectRatio: true,
supportsResolution: true,
},
geometry: {
sizes: [...SUPPORTED_SIZES],
aspectRatios: [...SUPPORTED_ASPECT_RATIOS],
resolutions: ["1K", "2K", "4K"],
},
output: {
formats: ["png", "jpeg", "webp"],
qualities: ["low", "medium", "high", "auto"],
},
},
async generateImage(req: ImageGenerationRequest): Promise<ImageGenerationResult> {
const pluginCfg = resolveImageConfig(req.cfg, getPluginConfig?.());
const explicitApiKey =
resolvePluginApiKey(pluginCfg?.apiKey) ??
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ??
normalizeOptionalString(process.env.BIFROST_API_KEY) ??
normalizeOptionalString(process.env.CCLOAD_API_KEY) ??
normalizeOptionalString(process.env.OPENAI_API_KEY);
let apiKey = explicitApiKey;
if (!apiKey) {
const { resolveApiKeyForProvider } = await import(
"openclaw/plugin-sdk/provider-auth-runtime"
);
const auth = await resolveApiKeyForProvider({
provider: "openai",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
modelApi: "openai-responses",
});
if (!auth.apiKey) {
throw new Error("OpenAI / Bifrost Image API key missing");
}
apiKey = auth.apiKey;
}
const [
{
assertOkOrThrowHttpError,
createProviderOperationDeadline,
createProviderOperationTimeoutResolver,
fetchWithTimeoutGuarded,
postJsonRequest,
postMultipartRequest,
readProviderJsonResponse,
resolveProviderHttpRequestConfig,
resolveProviderOperationTimeoutMs,
sanitizeConfiguredModelProviderRequest,
},
] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]);
const fetchFn = fetch;
const deadline = createProviderOperationDeadline({
timeoutMs: req.timeoutMs,
label: "OpenAI Image Generation",
});
const providerConfig = req.cfg?.models?.providers?.openai;
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
const requestConfig = sanitizeConfiguredModelProviderRequest({
...providerConfig?.request,
allowPrivateNetwork: allowPrivate,
});
const rawBaseUrl = resolveConfiguredImageBaseUrl(req.cfg, pluginCfg);
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: rawBaseUrl,
defaultBaseUrl: DEFAULT_OPENAI_IMAGE_BASE_URL,
allowPrivateNetwork: allowPrivate,
request: requestConfig,
defaultHeaders: {
Authorization: `Bearer ${apiKey}`,
},
provider: "openai",
capability: "image",
transport: "http",
});
const model =
normalizeOptionalString(req.model) ??
normalizeOptionalString(pluginCfg?.defaultModel) ??
DEFAULT_OPENAI_IMAGE_MODEL;
const size = resolveImageSize({
size: req.size,
aspectRatio: req.aspectRatio,
model,
});
const count = Math.max(1, Math.min(req.count ?? 1, 4));
const inputImages = req.inputImages ?? [];
const isEdit = inputImages.length > 0;
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
deadline,
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
});
let response: Response;
let release: () => Promise<void>;
if (isEdit) {
const form = new FormData();
form.set("model", model);
form.set("prompt", req.prompt);
form.set("n", String(count));
form.set("size", size);
form.set("response_format", "b64_json");
inputImages.forEach((img, idx) => {
const mime = img.mimeType || "image/png";
const ext = extensionForMime(mime)?.slice(1) || "png";
const name = img.fileName || `input-${idx + 1}.${ext}`;
form.append(
idx === 0 ? "image" : "image[]",
new File([bufferToBlobPart(img.buffer)], name, { type: mime }),
);
});
const multipartHeaders = new Headers(headers);
multipartHeaders.delete("Content-Type");
({ response, release } = await postMultipartRequest({
url: `${baseUrl}/images/edits`,
headers: multipartHeaders,
body: form,
timeoutMs: submitTimeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
}));
} else {
const jsonHeaders = new Headers(headers);
jsonHeaders.set("Content-Type", "application/json");
const jsonBody: Record<string, unknown> = {
model,
prompt: req.prompt,
n: count,
size,
response_format: "b64_json",
};
if (req.quality) {
jsonBody.quality = req.quality;
}
({ response, release } = await postJsonRequest({
url: `${baseUrl}/images/generations`,
headers: jsonHeaders,
body: jsonBody,
timeoutMs: submitTimeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
}));
}
interface OpenAIImageApiResponse {
created?: number;
data?: Array<{
b64_json?: string;
url?: string;
revised_prompt?: string;
}>;
error?: {
message?: string;
code?: string;
type?: string;
};
}
let payload: OpenAIImageApiResponse;
try {
await assertOkOrThrowHttpError(response, "Image generation request failed");
payload = await readProviderJsonResponse<OpenAIImageApiResponse>(
response,
"OpenAI image generation failed",
);
} finally {
await release();
}
if (payload.error?.message) {
throw new Error(`Image generation error: ${payload.error.message}`);
}
const dataItems = payload.data ?? [];
if (dataItems.length === 0) {
throw new Error("Image generation returned no image data");
}
const generatedImages: GeneratedImageAsset[] = [];
const downloadTimeout = createProviderOperationTimeoutResolver({
deadline,
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
});
for (let i = 0; i < dataItems.length; i++) {
const item = dataItems[i];
if (item.b64_json) {
const buffer = Buffer.from(item.b64_json, "base64");
generatedImages.push({
buffer,
mimeType: "image/png",
fileName: `image-${i + 1}.png`,
revisedPrompt: item.revised_prompt,
});
} else if (item.url) {
const dlResult = await fetchWithTimeoutGuarded(
item.url,
{ method: "GET" },
downloadTimeout(),
fetchFn,
{
allowPrivateNetwork,
auditContext: "openai-image-download",
},
);
try {
await assertOkOrThrowHttpError(dlResult.response, "Failed to download generated image");
const arrayBuf = await dlResult.response.arrayBuffer();
const mime = dlResult.response.headers.get("content-type") || "image/png";
const ext = extensionForMime(mime)?.slice(1) || "png";
generatedImages.push({
buffer: Buffer.from(arrayBuf),
mimeType: mime,
fileName: `image-${i + 1}.${ext}`,
revisedPrompt: item.revised_prompt,
});
} finally {
await dlResult.release();
}
}
}
if (generatedImages.length === 0) {
throw new Error("Failed to extract any valid image from provider response");
}
return {
images: generatedImages,
model,
metadata: {
created: payload.created,
size,
},
};
},
};
}
+8 -1
View File
@@ -7,9 +7,10 @@ describe("tools", () => {
expect(getToolPluginMetadata(entry)?.tools.map((tool) => tool.name)).toEqual(["echo"]);
});
it("registers both tool and video generation provider", () => {
it("registers tool, video generation provider, and image generation provider", () => {
const registeredTools: unknown[] = [];
const registeredVideoProviders: unknown[] = [];
const registeredImageProviders: unknown[] = [];
const mockApi = {
runtime: {
@@ -22,6 +23,9 @@ describe("tools", () => {
registerVideoGenerationProvider: vi.fn((provider) =>
registeredVideoProviders.push(provider),
),
registerImageGenerationProvider: vi.fn((provider) =>
registeredImageProviders.push(provider),
),
};
entry.register(mockApi as any);
@@ -31,5 +35,8 @@ describe("tools", () => {
expect(mockApi.registerVideoGenerationProvider).toHaveBeenCalledTimes(1);
expect(registeredVideoProviders[0]).toMatchObject({ id: "openai-video" });
expect(mockApi.registerImageGenerationProvider).toHaveBeenCalledTimes(1);
expect(registeredImageProviders[0]).toMatchObject({ id: "openai-image" });
});
});
+11
View File
@@ -3,12 +3,14 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin";
import { echoTool } from "./tools/echo.js";
import { buildOpenAIVideoGenerationProvider } from "./video/index.js";
import { buildOpenAIImageGenerationProvider } from "./image/index.js";
const entry = definePluginEntry({
id: "tools",
name: "Tools",
description: "Custom Tools and Media Providers for OpenClaw.",
register(api) {
// 1. Register tools
api.registerTool({
name: echoTool.name,
label: echoTool.label,
@@ -20,12 +22,21 @@ const entry = definePluginEntry({
},
});
// 2. Register video generation provider
api.registerVideoGenerationProvider(
buildOpenAIVideoGenerationProvider({
isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured,
getPluginConfig: () => api.pluginConfig,
}),
);
// 3. Register image generation provider
api.registerImageGenerationProvider(
buildOpenAIImageGenerationProvider({
isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured,
getPluginConfig: () => api.pluginConfig,
}),
);
},
});
+53 -1
View File
@@ -30,6 +30,58 @@ declare module "openclaw/plugin-sdk/video-generation" {
}
}
declare module "openclaw/plugin-sdk/image-generation" {
export interface GeneratedImageAsset {
buffer: Buffer;
mimeType: string;
fileName?: string;
revisedPrompt?: string;
metadata?: Record<string, unknown>;
}
export interface ImageGenerationRequest {
prompt: string;
model?: string;
count?: number;
size?: string;
aspectRatio?: string;
resolution?: string;
quality?: string;
outputFormat?: string;
background?: string;
inputImages?: Array<{ buffer: Buffer; mimeType?: string; fileName?: string }>;
cfg?: any;
agentDir?: string;
authStore?: any;
timeoutMs?: number;
ssrfPolicy?: any;
providerOptions?: Record<string, unknown>;
}
export interface ImageGenerationResult {
images: GeneratedImageAsset[];
model?: string;
metadata?: Record<string, unknown>;
}
export interface ImageGenerationProvider {
id: string;
aliases?: string[];
label?: string;
defaultModel?: string;
defaultTimeoutMs?: number;
models?: string[];
capabilities: any;
isConfigured?: (ctx: { cfg?: any }) => boolean;
generateImage: (req: ImageGenerationRequest) => Promise<ImageGenerationResult>;
}
export function parseOpenAiCompatibleImageResponse(
payload: unknown,
options?: { defaultMimeType?: string; malformedResponseError?: string; fileNamePrefix?: string; sniffMimeType?: boolean },
): GeneratedImageAsset[];
}
declare module "openclaw/plugin-sdk/provider-http" {
export function createProviderOperationDeadline(params: { timeoutMs?: number; label: string }): any;
export function createProviderOperationTimeoutResolver(params: { deadline: any; defaultTimeoutMs: number }): () => number;
@@ -39,7 +91,7 @@ declare module "openclaw/plugin-sdk/provider-http" {
export function assertOkOrThrowHttpError(response: Response, message: string): Promise<void>;
export function postMultipartRequest(params: any): Promise<{ response: Response; release: () => Promise<void> }>;
export function postJsonRequest(params: any): Promise<{ response: Response; release: () => Promise<void> }>;
export function readProviderJsonResponse<T = any>(response: Response, message: string): Promise<T>;
export function readProviderJsonResponse<T = any>(response: Response, message: string, opts?: any): Promise<T>;
export function pollProviderOperationJson<T = any>(params: any): Promise<T>;
export function resolveProviderOperationTimeoutMs(params: any): number;
export function resolveProviderHttpRequestConfig(params: any): any;