feat(image): add OpenAI/Bifrost compatible image generation provider
This commit is contained in:
1 parent
93ae5b7342
commit
2011077fcd
10 files changed
+1019
-21
No files matched your search
@@ -11,6 +11,7 @@ OpenClaw 统一私有扩展插件集合,聚合了自定义 Agent 工具、媒
|
||||
| 模块类别 | 标识符 (ID) | 契约类型 | 说明 | 详细文档 |
|
||||
| :--- | :--- | :--- | :--- | :--- |
|
||||
| **视频生成** | `openai-video` | `videoGenerationProviders` | OpenAI `/v1/videos` 协议兼容视频生成驱动(支持自建网关 / gemini-web2api / Sora 协议) | [查看文档 →](src/video/README.md) |
|
||||
| **图像生成** | `openai-image` | `imageGenerationProviders` | OpenAI `/v1/images` 协议兼容图像生成驱动(支持 Bifrost / ccLoad,修复 16:9 尺寸 Bug 与 Base64/URL 双模解析) | [查看文档 →](src/image/README.md) |
|
||||
| **基础工具** | `echo` | `tools` | 基础测试与连通性回显工具 | [查看文档 →](src/tools/README.md) |
|
||||
| **内置技能** | `agent-access` | `skills` | Bitwarden Agent Access 自动化技能 | [查看技能 →](skills/agent-access/SKILL.md) |
|
||||
|
||||
@@ -57,13 +58,16 @@ openclaw-plugin-tools/
|
||||
│ ├── types.d.ts # Plugin SDK 补充类型定义
|
||||
│ ├── tools/ # 工具模块集合
|
||||
│ │ ├── echo.ts
|
||||
│ │ └── README.md # 工具专属说明
|
||||
│ └── video/ # 视频生成驱动模块
|
||||
│ │ └── README.md # 📍 工具专属说明
|
||||
│ ├── video/ # 视频生成驱动模块
|
||||
│ │ ├── index.ts
|
||||
│ │ └── README.md # 📍 视频驱动专属说明
|
||||
│ └── image/ # 图像生成驱动模块
|
||||
│ ├── index.ts
|
||||
│ └── README.md # 视频驱动专属说明
|
||||
│ └── README.md # 📍 图像驱动专属说明
|
||||
└── skills/ # 官方规范就近内置技能
|
||||
└── agent-access/
|
||||
└── SKILL.md # 技能自描述文件
|
||||
└── SKILL.md # 📍 技能自描述文件
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { ImageGenerationProvider, ImageGenerationRequest } from "openclaw/plugin-sdk/image-generation";
|
||||
export interface SecretRefObject {
|
||||
source: string;
|
||||
provider: string;
|
||||
id: string;
|
||||
}
|
||||
export interface ImagePluginConfig {
|
||||
baseUrl?: string;
|
||||
apiKey?: string | SecretRefObject;
|
||||
allowPrivateNetwork?: boolean;
|
||||
defaultModel?: string;
|
||||
models?: string[];
|
||||
}
|
||||
export declare function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined;
|
||||
export declare function resolveImageConfig(cfg: ImageGenerationRequest["cfg"] | undefined, apiPluginConfig?: Record<string, unknown>): ImagePluginConfig | undefined;
|
||||
export declare function resolveConfiguredImageBaseUrl(cfg: ImageGenerationRequest["cfg"], pluginCfg?: ImagePluginConfig): string;
|
||||
export declare function resolveImageSize(params: {
|
||||
size?: string;
|
||||
aspectRatio?: string;
|
||||
model?: string;
|
||||
}): string;
|
||||
export declare function buildOpenAIImageGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }: {
|
||||
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
|
||||
getPluginConfig?: () => Record<string, unknown> | undefined;
|
||||
}): ImageGenerationProvider;
|
||||
Vendored
+316
@@ -0,0 +1,316 @@
|
||||
import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime";
|
||||
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
const DEFAULT_OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
|
||||
const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-2";
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const DEFAULT_SIZE = "1024x1024";
|
||||
const SUPPORTED_SIZES = [
|
||||
"1024x1024",
|
||||
"1536x1024",
|
||||
"1024x1536",
|
||||
"1792x1024",
|
||||
"1024x1792",
|
||||
"2048x2048",
|
||||
"2048x1152",
|
||||
"3840x2160",
|
||||
];
|
||||
const SUPPORTED_ASPECT_RATIOS = [
|
||||
"1:1",
|
||||
"16:9",
|
||||
"9:16",
|
||||
"4:3",
|
||||
"3:4",
|
||||
"3:2",
|
||||
"2:3",
|
||||
"21:9",
|
||||
];
|
||||
export function resolvePluginApiKey(rawKey) {
|
||||
if (typeof rawKey === "string") {
|
||||
return normalizeOptionalString(rawKey);
|
||||
}
|
||||
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
|
||||
return normalizeOptionalString(process.env[rawKey.id]);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
export function resolveImageConfig(cfg, apiPluginConfig) {
|
||||
const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config);
|
||||
const legacyConfig = (cfg?.plugins?.entries?.["openai-image"]?.config ??
|
||||
cfg?.plugins?.entries?.["bifrost-image"]?.config);
|
||||
const directImageConfig = toolsConfig?.image;
|
||||
const flatToolsConfig = toolsConfig?.baseUrl
|
||||
? {
|
||||
baseUrl: toolsConfig.baseUrl,
|
||||
apiKey: toolsConfig.apiKey,
|
||||
allowPrivateNetwork: toolsConfig.allowPrivateNetwork,
|
||||
defaultModel: toolsConfig.defaultModel,
|
||||
models: toolsConfig.models,
|
||||
}
|
||||
: undefined;
|
||||
return {
|
||||
...legacyConfig,
|
||||
...flatToolsConfig,
|
||||
...directImageConfig,
|
||||
};
|
||||
}
|
||||
export function resolveConfiguredImageBaseUrl(cfg, pluginCfg) {
|
||||
return (normalizeOptionalString(pluginCfg?.baseUrl) ??
|
||||
normalizeOptionalString(process.env.OPENAI_IMAGE_BASE_URL) ??
|
||||
normalizeOptionalString(process.env.BIFROST_BASE_URL) ??
|
||||
normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ??
|
||||
normalizeOptionalString(process.env.OPENAI_BASE_URL) ??
|
||||
DEFAULT_OPENAI_IMAGE_BASE_URL);
|
||||
}
|
||||
export function resolveImageSize(params) {
|
||||
const explicitSize = normalizeOptionalString(params.size);
|
||||
if (explicitSize) {
|
||||
return explicitSize;
|
||||
}
|
||||
// Safe aspect ratio normalization preventing issue #168665 (proxy rejecting 2048x1152)
|
||||
switch (normalizeOptionalString(params.aspectRatio)) {
|
||||
case "1:1":
|
||||
return "1024x1024";
|
||||
case "16:9":
|
||||
return "1792x1024";
|
||||
case "9:16":
|
||||
return "1024x1792";
|
||||
case "4:3":
|
||||
case "3:2":
|
||||
return "1536x1024";
|
||||
case "3:4":
|
||||
case "2:3":
|
||||
return "1024x1536";
|
||||
case "21:9":
|
||||
return "1792x1024";
|
||||
default:
|
||||
return DEFAULT_SIZE;
|
||||
}
|
||||
}
|
||||
export function buildOpenAIImageGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }) {
|
||||
return {
|
||||
id: "openai-image",
|
||||
aliases: ["bifrost-image", "custom-image"],
|
||||
label: "OpenAI Image (Custom / Bifrost)",
|
||||
defaultModel: DEFAULT_OPENAI_IMAGE_MODEL,
|
||||
models: [DEFAULT_OPENAI_IMAGE_MODEL, "gpt-image-1.5", "gemini-3.1-flash-image"],
|
||||
isConfigured: (ctx) => {
|
||||
const pluginCfg = resolveImageConfig(ctx.cfg, getPluginConfig?.());
|
||||
if (resolvePluginApiKey(pluginCfg?.apiKey) ||
|
||||
Boolean(pluginCfg?.apiKey) ||
|
||||
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ||
|
||||
normalizeOptionalString(process.env.BIFROST_API_KEY) ||
|
||||
normalizeOptionalString(process.env.CCLOAD_API_KEY) ||
|
||||
normalizeOptionalString(process.env.OPENAI_API_KEY)) {
|
||||
return true;
|
||||
}
|
||||
return isProviderApiKeyConfigured({
|
||||
provider: "openai",
|
||||
...ctx,
|
||||
profileTypes: ["api_key"],
|
||||
});
|
||||
},
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 4,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 4,
|
||||
maxInputImages: 5,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
geometry: {
|
||||
sizes: [...SUPPORTED_SIZES],
|
||||
aspectRatios: [...SUPPORTED_ASPECT_RATIOS],
|
||||
resolutions: ["1K", "2K", "4K"],
|
||||
},
|
||||
output: {
|
||||
formats: ["png", "jpeg", "webp"],
|
||||
qualities: ["low", "medium", "high", "auto"],
|
||||
},
|
||||
},
|
||||
async generateImage(req) {
|
||||
const pluginCfg = resolveImageConfig(req.cfg, getPluginConfig?.());
|
||||
const explicitApiKey = resolvePluginApiKey(pluginCfg?.apiKey) ??
|
||||
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ??
|
||||
normalizeOptionalString(process.env.BIFROST_API_KEY) ??
|
||||
normalizeOptionalString(process.env.CCLOAD_API_KEY) ??
|
||||
normalizeOptionalString(process.env.OPENAI_API_KEY);
|
||||
let apiKey = explicitApiKey;
|
||||
if (!apiKey) {
|
||||
const { resolveApiKeyForProvider } = await import("openclaw/plugin-sdk/provider-auth-runtime");
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "openai",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
modelApi: "openai-responses",
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("OpenAI / Bifrost Image API key missing");
|
||||
}
|
||||
apiKey = auth.apiKey;
|
||||
}
|
||||
const [{ assertOkOrThrowHttpError, createProviderOperationDeadline, createProviderOperationTimeoutResolver, fetchWithTimeoutGuarded, postJsonRequest, postMultipartRequest, readProviderJsonResponse, resolveProviderHttpRequestConfig, resolveProviderOperationTimeoutMs, sanitizeConfiguredModelProviderRequest, },] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]);
|
||||
const fetchFn = fetch;
|
||||
const deadline = createProviderOperationDeadline({
|
||||
timeoutMs: req.timeoutMs,
|
||||
label: "OpenAI Image Generation",
|
||||
});
|
||||
const providerConfig = req.cfg?.models?.providers?.openai;
|
||||
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
|
||||
const requestConfig = sanitizeConfiguredModelProviderRequest({
|
||||
...providerConfig?.request,
|
||||
allowPrivateNetwork: allowPrivate,
|
||||
});
|
||||
const rawBaseUrl = resolveConfiguredImageBaseUrl(req.cfg, pluginCfg);
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({
|
||||
baseUrl: rawBaseUrl,
|
||||
defaultBaseUrl: DEFAULT_OPENAI_IMAGE_BASE_URL,
|
||||
allowPrivateNetwork: allowPrivate,
|
||||
request: requestConfig,
|
||||
defaultHeaders: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
provider: "openai",
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
});
|
||||
const model = normalizeOptionalString(req.model) ??
|
||||
normalizeOptionalString(pluginCfg?.defaultModel) ??
|
||||
DEFAULT_OPENAI_IMAGE_MODEL;
|
||||
const size = resolveImageSize({
|
||||
size: req.size,
|
||||
aspectRatio: req.aspectRatio,
|
||||
model,
|
||||
});
|
||||
const count = Math.max(1, Math.min(req.count ?? 1, 4));
|
||||
const inputImages = req.inputImages ?? [];
|
||||
const isEdit = inputImages.length > 0;
|
||||
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
});
|
||||
let response;
|
||||
let release;
|
||||
if (isEdit) {
|
||||
const form = new FormData();
|
||||
form.set("model", model);
|
||||
form.set("prompt", req.prompt);
|
||||
form.set("n", String(count));
|
||||
form.set("size", size);
|
||||
form.set("response_format", "b64_json");
|
||||
inputImages.forEach((img, idx) => {
|
||||
const mime = img.mimeType || "image/png";
|
||||
const ext = extensionForMime(mime)?.slice(1) || "png";
|
||||
const name = img.fileName || `input-${idx + 1}.${ext}`;
|
||||
form.append(idx === 0 ? "image" : "image[]", new File([bufferToBlobPart(img.buffer)], name, { type: mime }));
|
||||
});
|
||||
const multipartHeaders = new Headers(headers);
|
||||
multipartHeaders.delete("Content-Type");
|
||||
({ response, release } = await postMultipartRequest({
|
||||
url: `${baseUrl}/images/edits`,
|
||||
headers: multipartHeaders,
|
||||
body: form,
|
||||
timeoutMs: submitTimeoutMs,
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
}));
|
||||
}
|
||||
else {
|
||||
const jsonHeaders = new Headers(headers);
|
||||
jsonHeaders.set("Content-Type", "application/json");
|
||||
const jsonBody = {
|
||||
model,
|
||||
prompt: req.prompt,
|
||||
n: count,
|
||||
size,
|
||||
response_format: "b64_json",
|
||||
};
|
||||
if (req.quality) {
|
||||
jsonBody.quality = req.quality;
|
||||
}
|
||||
({ response, release } = await postJsonRequest({
|
||||
url: `${baseUrl}/images/generations`,
|
||||
headers: jsonHeaders,
|
||||
body: jsonBody,
|
||||
timeoutMs: submitTimeoutMs,
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
}));
|
||||
}
|
||||
let payload;
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "Image generation request failed");
|
||||
payload = await readProviderJsonResponse(response, "OpenAI image generation failed");
|
||||
}
|
||||
finally {
|
||||
await release();
|
||||
}
|
||||
if (payload.error?.message) {
|
||||
throw new Error(`Image generation error: ${payload.error.message}`);
|
||||
}
|
||||
const dataItems = payload.data ?? [];
|
||||
if (dataItems.length === 0) {
|
||||
throw new Error("Image generation returned no image data");
|
||||
}
|
||||
const generatedImages = [];
|
||||
const downloadTimeout = createProviderOperationTimeoutResolver({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
});
|
||||
for (let i = 0; i < dataItems.length; i++) {
|
||||
const item = dataItems[i];
|
||||
if (item.b64_json) {
|
||||
const buffer = Buffer.from(item.b64_json, "base64");
|
||||
generatedImages.push({
|
||||
buffer,
|
||||
mimeType: "image/png",
|
||||
fileName: `image-${i + 1}.png`,
|
||||
revisedPrompt: item.revised_prompt,
|
||||
});
|
||||
}
|
||||
else if (item.url) {
|
||||
const dlResult = await fetchWithTimeoutGuarded(item.url, { method: "GET" }, downloadTimeout(), fetchFn, {
|
||||
allowPrivateNetwork,
|
||||
auditContext: "openai-image-download",
|
||||
});
|
||||
try {
|
||||
await assertOkOrThrowHttpError(dlResult.response, "Failed to download generated image");
|
||||
const arrayBuf = await dlResult.response.arrayBuffer();
|
||||
const mime = dlResult.response.headers.get("content-type") || "image/png";
|
||||
const ext = extensionForMime(mime)?.slice(1) || "png";
|
||||
generatedImages.push({
|
||||
buffer: Buffer.from(arrayBuf),
|
||||
mimeType: mime,
|
||||
fileName: `image-${i + 1}.${ext}`,
|
||||
revisedPrompt: item.revised_prompt,
|
||||
});
|
||||
}
|
||||
finally {
|
||||
await dlResult.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (generatedImages.length === 0) {
|
||||
throw new Error("Failed to extract any valid image from provider response");
|
||||
}
|
||||
return {
|
||||
images: generatedImages,
|
||||
model,
|
||||
metadata: {
|
||||
created: payload.created,
|
||||
size,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
Vendored
+8
@@ -3,11 +3,13 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin";
|
||||
import { echoTool } from "./tools/echo.js";
|
||||
import { buildOpenAIVideoGenerationProvider } from "./video/index.js";
|
||||
import { buildOpenAIImageGenerationProvider } from "./image/index.js";
|
||||
const entry = definePluginEntry({
|
||||
id: "tools",
|
||||
name: "Tools",
|
||||
description: "Custom Tools and Media Providers for OpenClaw.",
|
||||
register(api) {
|
||||
// 1. Register tools
|
||||
api.registerTool({
|
||||
name: echoTool.name,
|
||||
label: echoTool.label,
|
||||
@@ -18,10 +20,16 @@ const entry = definePluginEntry({
|
||||
return jsonResult(result);
|
||||
},
|
||||
});
|
||||
// 2. Register video generation provider
|
||||
api.registerVideoGenerationProvider(buildOpenAIVideoGenerationProvider({
|
||||
isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured,
|
||||
getPluginConfig: () => api.pluginConfig,
|
||||
}));
|
||||
// 3. Register image generation provider
|
||||
api.registerImageGenerationProvider(buildOpenAIImageGenerationProvider({
|
||||
isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured,
|
||||
getPluginConfig: () => api.pluginConfig,
|
||||
}));
|
||||
},
|
||||
});
|
||||
Object.defineProperty(entry, toolPluginMetadataSymbol, {
|
||||
|
||||
+69
-15
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"id": "tools",
|
||||
"name": "Tools",
|
||||
"description": "Custom Tools and Media Providers for OpenClaw.",
|
||||
"version": "0.2.0",
|
||||
"description": "Custom Tools and Media Providers (Video & Image) for OpenClaw.",
|
||||
"version": "0.3.0",
|
||||
"categories": [
|
||||
"media",
|
||||
"utilities"
|
||||
@@ -23,6 +23,15 @@
|
||||
"BIFROST_API_KEY",
|
||||
"OPENAI_API_KEY"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "openai-image",
|
||||
"envVars": [
|
||||
"OPENAI_IMAGE_API_KEY",
|
||||
"BIFROST_API_KEY",
|
||||
"CCLOAD_API_KEY",
|
||||
"OPENAI_API_KEY"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -32,6 +41,9 @@
|
||||
],
|
||||
"videoGenerationProviders": [
|
||||
"openai-video"
|
||||
],
|
||||
"imageGenerationProviders": [
|
||||
"openai-image"
|
||||
]
|
||||
},
|
||||
"videoGenerationProviderMetadata": {
|
||||
@@ -43,6 +55,15 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"imageGenerationProviderMetadata": {
|
||||
"openai-image": {
|
||||
"authSignals": [
|
||||
{
|
||||
"provider": "openai"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"configContracts": {
|
||||
"secretInputs": {
|
||||
"paths": [
|
||||
@@ -55,6 +76,11 @@
|
||||
"path": "video.apiKey",
|
||||
"expected": "string",
|
||||
"ownerKind": "capability"
|
||||
},
|
||||
{
|
||||
"path": "image.apiKey",
|
||||
"expected": "string",
|
||||
"ownerKind": "capability"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -67,6 +93,10 @@
|
||||
"video.apiKey": {
|
||||
"label": "Video API Key",
|
||||
"sensitive": true
|
||||
},
|
||||
"image.apiKey": {
|
||||
"label": "Image API Key",
|
||||
"sensitive": true
|
||||
}
|
||||
},
|
||||
"configSchema": {
|
||||
@@ -76,7 +106,7 @@
|
||||
"baseUrl": {
|
||||
"type": "string",
|
||||
"title": "API Base URL",
|
||||
"description": "Base URL of the OpenAI-compatible video endpoint"
|
||||
"description": "Base URL of the OpenAI-compatible endpoint"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": [
|
||||
@@ -84,7 +114,7 @@
|
||||
"object"
|
||||
],
|
||||
"title": "API Key",
|
||||
"description": "API Key or SecretRef for video generation"
|
||||
"description": "API Key or SecretRef"
|
||||
},
|
||||
"allowPrivateNetwork": {
|
||||
"type": "boolean",
|
||||
@@ -94,27 +124,20 @@
|
||||
},
|
||||
"defaultModel": {
|
||||
"type": "string",
|
||||
"title": "Default Video Model",
|
||||
"description": "Default model ID to use when no model is explicitly requested",
|
||||
"default": "sora-2"
|
||||
"title": "Default Model",
|
||||
"description": "Default model ID"
|
||||
},
|
||||
"models": {
|
||||
"type": "array",
|
||||
"title": "Available Models",
|
||||
"description": "List of available video model IDs",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"default": [
|
||||
"sora-2",
|
||||
"sora-2-pro",
|
||||
"gemini-video"
|
||||
]
|
||||
}
|
||||
},
|
||||
"video": {
|
||||
"type": "object",
|
||||
"title": "Video Generation Settings",
|
||||
"description": "Nested configuration for video generation",
|
||||
"description": "Configuration for video generation (openai-video)",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"baseUrl": {
|
||||
@@ -141,6 +164,37 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"image": {
|
||||
"type": "object",
|
||||
"title": "Image Generation Settings",
|
||||
"description": "Configuration for image generation (openai-image / bifrost-image)",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"baseUrl": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": [
|
||||
"string",
|
||||
"object"
|
||||
]
|
||||
},
|
||||
"allowPrivateNetwork": {
|
||||
"type": "boolean",
|
||||
"default": true
|
||||
},
|
||||
"defaultModel": {
|
||||
"type": "string",
|
||||
"default": "gpt-image-2"
|
||||
},
|
||||
"models": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# OpenAI Image Provider (`openai-image` / `bifrost-image`)
|
||||
|
||||
OpenClaw 图像生成驱动模块,适配标准 **OpenAI Images API**(`/v1/images/generations` 及 `/v1/images/edits` 规范)。
|
||||
|
||||
专为自建聚合网关或第三方中转(如 `Bifrost`、`ccLoad`)定制打造:
|
||||
* **尺寸安全归一化**:修复官方驱动在反代环境下因 `16:9` 映射为 `2048x1152` 导致上游报 400 错误的 Bug(Issue #168665),自动将 `16:9` / `9:16` 适配为上游通用的安全比例规格(如 `1792x1024` / `1536x1024`)。
|
||||
* **格式双模自适应**:同时支持解析上游返回的 `b64_json`(Base64 内存直解码)与 `url`(自动安全下载并保存为本地 Buffer),彻底杜绝 `response malformed` 解析失败。
|
||||
* **图生图编辑支持**:当传入 `inputImages` 时,自动走 Multipart 表单发起编辑请求;纯文本生图时走轻量 JSON 请求。
|
||||
* **私有网络与反代直通**:原生支持私有/内网端点请求(`allowPrivateNetwork: true`),兼容 `127.0.0.1`、`host.incus.internal` 或自建反代网关。
|
||||
|
||||
---
|
||||
|
||||
## 标识符与别名
|
||||
|
||||
* **Provider ID**:`openai-image`
|
||||
* **支持别名 (Aliases)**:`bifrost-image`, `custom-image`
|
||||
* **默认模型**:`gpt-image-2`
|
||||
* **支持模型列表**:`gpt-image-2`, `gpt-image-1.5`, `gemini-3.1-flash-image`(也可在配置中自定义扩展)
|
||||
|
||||
---
|
||||
|
||||
## 配置说明
|
||||
|
||||
支持通过 **Web 控制台(Control UI)** 或直接编辑 `~/.openclaw/openclaw.json`。
|
||||
|
||||
### 方式 1:Web 控制台可视化配置
|
||||
进入 **Settings → Plugins → Tools**:
|
||||
* **Image Base URL**:填写图像上游端点(例如 `https://Direct-Bifrost.oo1.dev/v1` 或内部反代地址)
|
||||
* **Image API Key**:填写 API Key 或 SecretRef(留空时自动回退读取环境变量 `OPENAI_IMAGE_API_KEY`、`BIFROST_API_KEY`、`CCLOAD_API_KEY` 或 `OPENAI_API_KEY`)
|
||||
* **Default Image Model**:例如 `gpt-image-2`
|
||||
|
||||
### 方式 2:编辑 `openclaw.json`
|
||||
|
||||
在 `~/.openclaw/openclaw.json` 中配置:
|
||||
|
||||
```json5
|
||||
{
|
||||
"plugins": {
|
||||
"entries": {
|
||||
"tools": {
|
||||
"enabled": true,
|
||||
"config": {
|
||||
"image": {
|
||||
"baseUrl": "https://Direct-Bifrost.oo1.dev/v1",
|
||||
"apiKey": "sk-your-bifrost-key",
|
||||
"defaultModel": "gpt-image-2",
|
||||
"allowPrivateNetwork": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"agents": {
|
||||
"defaults": {
|
||||
"mediaModels": {
|
||||
"image": {
|
||||
"primary": "openai-image/gpt-image-2",
|
||||
"fallbacks": [
|
||||
"openai-image/gemini-3.1-flash-image"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 使用方法
|
||||
|
||||
配置完成后,OpenClaw 内置的 `image_generate` 工具即可自动调度:
|
||||
* **文本生成图像**:
|
||||
> “生成一张 16:9 的秋日晨光现代书房桌面壁纸”
|
||||
* **垫图修改/编辑**:
|
||||
在对话中附带图片,提示助手基于该图进行风格调整或重绘。
|
||||
@@ -0,0 +1,444 @@
|
||||
import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime";
|
||||
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
GeneratedImageAsset,
|
||||
ImageGenerationProvider,
|
||||
ImageGenerationRequest,
|
||||
ImageGenerationResult,
|
||||
} from "openclaw/plugin-sdk/image-generation";
|
||||
|
||||
const DEFAULT_OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
|
||||
const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-2";
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const DEFAULT_SIZE = "1024x1024";
|
||||
|
||||
const SUPPORTED_SIZES = [
|
||||
"1024x1024",
|
||||
"1536x1024",
|
||||
"1024x1536",
|
||||
"1792x1024",
|
||||
"1024x1792",
|
||||
"2048x2048",
|
||||
"2048x1152",
|
||||
"3840x2160",
|
||||
] as const;
|
||||
|
||||
const SUPPORTED_ASPECT_RATIOS = [
|
||||
"1:1",
|
||||
"16:9",
|
||||
"9:16",
|
||||
"4:3",
|
||||
"3:4",
|
||||
"3:2",
|
||||
"2:3",
|
||||
"21:9",
|
||||
] as const;
|
||||
|
||||
export interface SecretRefObject {
|
||||
source: string;
|
||||
provider: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface ImagePluginConfig {
|
||||
baseUrl?: string;
|
||||
apiKey?: string | SecretRefObject;
|
||||
allowPrivateNetwork?: boolean;
|
||||
defaultModel?: string;
|
||||
models?: string[];
|
||||
}
|
||||
|
||||
export function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined {
|
||||
if (typeof rawKey === "string") {
|
||||
return normalizeOptionalString(rawKey);
|
||||
}
|
||||
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
|
||||
return normalizeOptionalString(process.env[rawKey.id]);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveImageConfig(
|
||||
cfg: ImageGenerationRequest["cfg"] | undefined,
|
||||
apiPluginConfig?: Record<string, unknown>,
|
||||
): ImagePluginConfig | undefined {
|
||||
const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config) as
|
||||
| {
|
||||
image?: ImagePluginConfig;
|
||||
baseUrl?: string;
|
||||
apiKey?: string | SecretRefObject;
|
||||
defaultModel?: string;
|
||||
models?: string[];
|
||||
allowPrivateNetwork?: boolean;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
const legacyConfig = (
|
||||
cfg?.plugins?.entries?.["openai-image"]?.config ??
|
||||
cfg?.plugins?.entries?.["bifrost-image"]?.config
|
||||
) as ImagePluginConfig | undefined;
|
||||
|
||||
const directImageConfig = toolsConfig?.image;
|
||||
const flatToolsConfig: ImagePluginConfig | undefined = toolsConfig?.baseUrl
|
||||
? {
|
||||
baseUrl: toolsConfig.baseUrl,
|
||||
apiKey: toolsConfig.apiKey,
|
||||
allowPrivateNetwork: toolsConfig.allowPrivateNetwork,
|
||||
defaultModel: toolsConfig.defaultModel,
|
||||
models: toolsConfig.models,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
...legacyConfig,
|
||||
...flatToolsConfig,
|
||||
...directImageConfig,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveConfiguredImageBaseUrl(
|
||||
cfg: ImageGenerationRequest["cfg"],
|
||||
pluginCfg?: ImagePluginConfig,
|
||||
): string {
|
||||
return (
|
||||
normalizeOptionalString(pluginCfg?.baseUrl) ??
|
||||
normalizeOptionalString(process.env.OPENAI_IMAGE_BASE_URL) ??
|
||||
normalizeOptionalString(process.env.BIFROST_BASE_URL) ??
|
||||
normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ??
|
||||
normalizeOptionalString(process.env.OPENAI_BASE_URL) ??
|
||||
DEFAULT_OPENAI_IMAGE_BASE_URL
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveImageSize(params: {
|
||||
size?: string;
|
||||
aspectRatio?: string;
|
||||
model?: string;
|
||||
}): string {
|
||||
const explicitSize = normalizeOptionalString(params.size);
|
||||
if (explicitSize) {
|
||||
return explicitSize;
|
||||
}
|
||||
|
||||
// Safe aspect ratio normalization preventing issue #168665 (proxy rejecting 2048x1152)
|
||||
switch (normalizeOptionalString(params.aspectRatio)) {
|
||||
case "1:1":
|
||||
return "1024x1024";
|
||||
case "16:9":
|
||||
return "1792x1024";
|
||||
case "9:16":
|
||||
return "1024x1792";
|
||||
case "4:3":
|
||||
case "3:2":
|
||||
return "1536x1024";
|
||||
case "3:4":
|
||||
case "2:3":
|
||||
return "1024x1536";
|
||||
case "21:9":
|
||||
return "1792x1024";
|
||||
default:
|
||||
return DEFAULT_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildOpenAIImageGenerationProvider({
|
||||
isProviderApiKeyConfigured,
|
||||
getPluginConfig,
|
||||
}: {
|
||||
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
|
||||
getPluginConfig?: () => Record<string, unknown> | undefined;
|
||||
}): ImageGenerationProvider {
|
||||
return {
|
||||
id: "openai-image",
|
||||
aliases: ["bifrost-image", "custom-image"],
|
||||
label: "OpenAI Image (Custom / Bifrost)",
|
||||
defaultModel: DEFAULT_OPENAI_IMAGE_MODEL,
|
||||
models: [DEFAULT_OPENAI_IMAGE_MODEL, "gpt-image-1.5", "gemini-3.1-flash-image"],
|
||||
isConfigured: (ctx) => {
|
||||
const pluginCfg = resolveImageConfig(ctx.cfg, getPluginConfig?.());
|
||||
if (
|
||||
resolvePluginApiKey(pluginCfg?.apiKey) ||
|
||||
Boolean(pluginCfg?.apiKey) ||
|
||||
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ||
|
||||
normalizeOptionalString(process.env.BIFROST_API_KEY) ||
|
||||
normalizeOptionalString(process.env.CCLOAD_API_KEY) ||
|
||||
normalizeOptionalString(process.env.OPENAI_API_KEY)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return isProviderApiKeyConfigured({
|
||||
provider: "openai",
|
||||
...ctx,
|
||||
profileTypes: ["api_key"],
|
||||
});
|
||||
},
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 4,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 4,
|
||||
maxInputImages: 5,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
geometry: {
|
||||
sizes: [...SUPPORTED_SIZES],
|
||||
aspectRatios: [...SUPPORTED_ASPECT_RATIOS],
|
||||
resolutions: ["1K", "2K", "4K"],
|
||||
},
|
||||
output: {
|
||||
formats: ["png", "jpeg", "webp"],
|
||||
qualities: ["low", "medium", "high", "auto"],
|
||||
},
|
||||
},
|
||||
async generateImage(req: ImageGenerationRequest): Promise<ImageGenerationResult> {
|
||||
const pluginCfg = resolveImageConfig(req.cfg, getPluginConfig?.());
|
||||
const explicitApiKey =
|
||||
resolvePluginApiKey(pluginCfg?.apiKey) ??
|
||||
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ??
|
||||
normalizeOptionalString(process.env.BIFROST_API_KEY) ??
|
||||
normalizeOptionalString(process.env.CCLOAD_API_KEY) ??
|
||||
normalizeOptionalString(process.env.OPENAI_API_KEY);
|
||||
|
||||
let apiKey = explicitApiKey;
|
||||
if (!apiKey) {
|
||||
const { resolveApiKeyForProvider } = await import(
|
||||
"openclaw/plugin-sdk/provider-auth-runtime"
|
||||
);
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "openai",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
modelApi: "openai-responses",
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("OpenAI / Bifrost Image API key missing");
|
||||
}
|
||||
apiKey = auth.apiKey;
|
||||
}
|
||||
|
||||
const [
|
||||
{
|
||||
assertOkOrThrowHttpError,
|
||||
createProviderOperationDeadline,
|
||||
createProviderOperationTimeoutResolver,
|
||||
fetchWithTimeoutGuarded,
|
||||
postJsonRequest,
|
||||
postMultipartRequest,
|
||||
readProviderJsonResponse,
|
||||
resolveProviderHttpRequestConfig,
|
||||
resolveProviderOperationTimeoutMs,
|
||||
sanitizeConfiguredModelProviderRequest,
|
||||
},
|
||||
] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]);
|
||||
|
||||
const fetchFn = fetch;
|
||||
const deadline = createProviderOperationDeadline({
|
||||
timeoutMs: req.timeoutMs,
|
||||
label: "OpenAI Image Generation",
|
||||
});
|
||||
|
||||
const providerConfig = req.cfg?.models?.providers?.openai;
|
||||
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
|
||||
const requestConfig = sanitizeConfiguredModelProviderRequest({
|
||||
...providerConfig?.request,
|
||||
allowPrivateNetwork: allowPrivate,
|
||||
});
|
||||
|
||||
const rawBaseUrl = resolveConfiguredImageBaseUrl(req.cfg, pluginCfg);
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveProviderHttpRequestConfig({
|
||||
baseUrl: rawBaseUrl,
|
||||
defaultBaseUrl: DEFAULT_OPENAI_IMAGE_BASE_URL,
|
||||
allowPrivateNetwork: allowPrivate,
|
||||
request: requestConfig,
|
||||
defaultHeaders: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
provider: "openai",
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
});
|
||||
|
||||
const model =
|
||||
normalizeOptionalString(req.model) ??
|
||||
normalizeOptionalString(pluginCfg?.defaultModel) ??
|
||||
DEFAULT_OPENAI_IMAGE_MODEL;
|
||||
|
||||
const size = resolveImageSize({
|
||||
size: req.size,
|
||||
aspectRatio: req.aspectRatio,
|
||||
model,
|
||||
});
|
||||
|
||||
const count = Math.max(1, Math.min(req.count ?? 1, 4));
|
||||
const inputImages = req.inputImages ?? [];
|
||||
const isEdit = inputImages.length > 0;
|
||||
|
||||
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
let response: Response;
|
||||
let release: () => Promise<void>;
|
||||
|
||||
if (isEdit) {
|
||||
const form = new FormData();
|
||||
form.set("model", model);
|
||||
form.set("prompt", req.prompt);
|
||||
form.set("n", String(count));
|
||||
form.set("size", size);
|
||||
form.set("response_format", "b64_json");
|
||||
|
||||
inputImages.forEach((img, idx) => {
|
||||
const mime = img.mimeType || "image/png";
|
||||
const ext = extensionForMime(mime)?.slice(1) || "png";
|
||||
const name = img.fileName || `input-${idx + 1}.${ext}`;
|
||||
form.append(
|
||||
idx === 0 ? "image" : "image[]",
|
||||
new File([bufferToBlobPart(img.buffer)], name, { type: mime }),
|
||||
);
|
||||
});
|
||||
|
||||
const multipartHeaders = new Headers(headers);
|
||||
multipartHeaders.delete("Content-Type");
|
||||
|
||||
({ response, release } = await postMultipartRequest({
|
||||
url: `${baseUrl}/images/edits`,
|
||||
headers: multipartHeaders,
|
||||
body: form,
|
||||
timeoutMs: submitTimeoutMs,
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
}));
|
||||
} else {
|
||||
const jsonHeaders = new Headers(headers);
|
||||
jsonHeaders.set("Content-Type", "application/json");
|
||||
|
||||
const jsonBody: Record<string, unknown> = {
|
||||
model,
|
||||
prompt: req.prompt,
|
||||
n: count,
|
||||
size,
|
||||
response_format: "b64_json",
|
||||
};
|
||||
|
||||
if (req.quality) {
|
||||
jsonBody.quality = req.quality;
|
||||
}
|
||||
|
||||
({ response, release } = await postJsonRequest({
|
||||
url: `${baseUrl}/images/generations`,
|
||||
headers: jsonHeaders,
|
||||
body: jsonBody,
|
||||
timeoutMs: submitTimeoutMs,
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
}));
|
||||
}
|
||||
|
||||
interface OpenAIImageApiResponse {
|
||||
created?: number;
|
||||
data?: Array<{
|
||||
b64_json?: string;
|
||||
url?: string;
|
||||
revised_prompt?: string;
|
||||
}>;
|
||||
error?: {
|
||||
message?: string;
|
||||
code?: string;
|
||||
type?: string;
|
||||
};
|
||||
}
|
||||
|
||||
let payload: OpenAIImageApiResponse;
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "Image generation request failed");
|
||||
payload = await readProviderJsonResponse<OpenAIImageApiResponse>(
|
||||
response,
|
||||
"OpenAI image generation failed",
|
||||
);
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
|
||||
if (payload.error?.message) {
|
||||
throw new Error(`Image generation error: ${payload.error.message}`);
|
||||
}
|
||||
|
||||
const dataItems = payload.data ?? [];
|
||||
if (dataItems.length === 0) {
|
||||
throw new Error("Image generation returned no image data");
|
||||
}
|
||||
|
||||
const generatedImages: GeneratedImageAsset[] = [];
|
||||
const downloadTimeout = createProviderOperationTimeoutResolver({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
for (let i = 0; i < dataItems.length; i++) {
|
||||
const item = dataItems[i];
|
||||
if (item.b64_json) {
|
||||
const buffer = Buffer.from(item.b64_json, "base64");
|
||||
generatedImages.push({
|
||||
buffer,
|
||||
mimeType: "image/png",
|
||||
fileName: `image-${i + 1}.png`,
|
||||
revisedPrompt: item.revised_prompt,
|
||||
});
|
||||
} else if (item.url) {
|
||||
const dlResult = await fetchWithTimeoutGuarded(
|
||||
item.url,
|
||||
{ method: "GET" },
|
||||
downloadTimeout(),
|
||||
fetchFn,
|
||||
{
|
||||
allowPrivateNetwork,
|
||||
auditContext: "openai-image-download",
|
||||
},
|
||||
);
|
||||
try {
|
||||
await assertOkOrThrowHttpError(dlResult.response, "Failed to download generated image");
|
||||
const arrayBuf = await dlResult.response.arrayBuffer();
|
||||
const mime = dlResult.response.headers.get("content-type") || "image/png";
|
||||
const ext = extensionForMime(mime)?.slice(1) || "png";
|
||||
generatedImages.push({
|
||||
buffer: Buffer.from(arrayBuf),
|
||||
mimeType: mime,
|
||||
fileName: `image-${i + 1}.${ext}`,
|
||||
revisedPrompt: item.revised_prompt,
|
||||
});
|
||||
} finally {
|
||||
await dlResult.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (generatedImages.length === 0) {
|
||||
throw new Error("Failed to extract any valid image from provider response");
|
||||
}
|
||||
|
||||
return {
|
||||
images: generatedImages,
|
||||
model,
|
||||
metadata: {
|
||||
created: payload.created,
|
||||
size,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
+8
-1
@@ -7,9 +7,10 @@ describe("tools", () => {
|
||||
expect(getToolPluginMetadata(entry)?.tools.map((tool) => tool.name)).toEqual(["echo"]);
|
||||
});
|
||||
|
||||
it("registers both tool and video generation provider", () => {
|
||||
it("registers tool, video generation provider, and image generation provider", () => {
|
||||
const registeredTools: unknown[] = [];
|
||||
const registeredVideoProviders: unknown[] = [];
|
||||
const registeredImageProviders: unknown[] = [];
|
||||
|
||||
const mockApi = {
|
||||
runtime: {
|
||||
@@ -22,6 +23,9 @@ describe("tools", () => {
|
||||
registerVideoGenerationProvider: vi.fn((provider) =>
|
||||
registeredVideoProviders.push(provider),
|
||||
),
|
||||
registerImageGenerationProvider: vi.fn((provider) =>
|
||||
registeredImageProviders.push(provider),
|
||||
),
|
||||
};
|
||||
|
||||
entry.register(mockApi as any);
|
||||
@@ -31,5 +35,8 @@ describe("tools", () => {
|
||||
|
||||
expect(mockApi.registerVideoGenerationProvider).toHaveBeenCalledTimes(1);
|
||||
expect(registeredVideoProviders[0]).toMatchObject({ id: "openai-video" });
|
||||
|
||||
expect(mockApi.registerImageGenerationProvider).toHaveBeenCalledTimes(1);
|
||||
expect(registeredImageProviders[0]).toMatchObject({ id: "openai-image" });
|
||||
});
|
||||
});
|
||||
@@ -3,12 +3,14 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin";
|
||||
import { echoTool } from "./tools/echo.js";
|
||||
import { buildOpenAIVideoGenerationProvider } from "./video/index.js";
|
||||
import { buildOpenAIImageGenerationProvider } from "./image/index.js";
|
||||
|
||||
const entry = definePluginEntry({
|
||||
id: "tools",
|
||||
name: "Tools",
|
||||
description: "Custom Tools and Media Providers for OpenClaw.",
|
||||
register(api) {
|
||||
// 1. Register tools
|
||||
api.registerTool({
|
||||
name: echoTool.name,
|
||||
label: echoTool.label,
|
||||
@@ -20,12 +22,21 @@ const entry = definePluginEntry({
|
||||
},
|
||||
});
|
||||
|
||||
// 2. Register video generation provider
|
||||
api.registerVideoGenerationProvider(
|
||||
buildOpenAIVideoGenerationProvider({
|
||||
isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured,
|
||||
getPluginConfig: () => api.pluginConfig,
|
||||
}),
|
||||
);
|
||||
|
||||
// 3. Register image generation provider
|
||||
api.registerImageGenerationProvider(
|
||||
buildOpenAIImageGenerationProvider({
|
||||
isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured,
|
||||
getPluginConfig: () => api.pluginConfig,
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
Vendored
+53
-1
@@ -30,6 +30,58 @@ declare module "openclaw/plugin-sdk/video-generation" {
|
||||
}
|
||||
}
|
||||
|
||||
declare module "openclaw/plugin-sdk/image-generation" {
|
||||
export interface GeneratedImageAsset {
|
||||
buffer: Buffer;
|
||||
mimeType: string;
|
||||
fileName?: string;
|
||||
revisedPrompt?: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ImageGenerationRequest {
|
||||
prompt: string;
|
||||
model?: string;
|
||||
count?: number;
|
||||
size?: string;
|
||||
aspectRatio?: string;
|
||||
resolution?: string;
|
||||
quality?: string;
|
||||
outputFormat?: string;
|
||||
background?: string;
|
||||
inputImages?: Array<{ buffer: Buffer; mimeType?: string; fileName?: string }>;
|
||||
cfg?: any;
|
||||
agentDir?: string;
|
||||
authStore?: any;
|
||||
timeoutMs?: number;
|
||||
ssrfPolicy?: any;
|
||||
providerOptions?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ImageGenerationResult {
|
||||
images: GeneratedImageAsset[];
|
||||
model?: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ImageGenerationProvider {
|
||||
id: string;
|
||||
aliases?: string[];
|
||||
label?: string;
|
||||
defaultModel?: string;
|
||||
defaultTimeoutMs?: number;
|
||||
models?: string[];
|
||||
capabilities: any;
|
||||
isConfigured?: (ctx: { cfg?: any }) => boolean;
|
||||
generateImage: (req: ImageGenerationRequest) => Promise<ImageGenerationResult>;
|
||||
}
|
||||
|
||||
export function parseOpenAiCompatibleImageResponse(
|
||||
payload: unknown,
|
||||
options?: { defaultMimeType?: string; malformedResponseError?: string; fileNamePrefix?: string; sniffMimeType?: boolean },
|
||||
): GeneratedImageAsset[];
|
||||
}
|
||||
|
||||
declare module "openclaw/plugin-sdk/provider-http" {
|
||||
export function createProviderOperationDeadline(params: { timeoutMs?: number; label: string }): any;
|
||||
export function createProviderOperationTimeoutResolver(params: { deadline: any; defaultTimeoutMs: number }): () => number;
|
||||
@@ -39,7 +91,7 @@ declare module "openclaw/plugin-sdk/provider-http" {
|
||||
export function assertOkOrThrowHttpError(response: Response, message: string): Promise<void>;
|
||||
export function postMultipartRequest(params: any): Promise<{ response: Response; release: () => Promise<void> }>;
|
||||
export function postJsonRequest(params: any): Promise<{ response: Response; release: () => Promise<void> }>;
|
||||
export function readProviderJsonResponse<T = any>(response: Response, message: string): Promise<T>;
|
||||
export function readProviderJsonResponse<T = any>(response: Response, message: string, opts?: any): Promise<T>;
|
||||
export function pollProviderOperationJson<T = any>(params: any): Promise<T>;
|
||||
export function resolveProviderOperationTimeoutMs(params: any): number;
|
||||
export function resolveProviderHttpRequestConfig(params: any): any;
|
||||
|
||||
Reference in new issue
Block a user