feat: split image providers into 1h-openai-image and 1h-gemini-image, rename video to 1h-openai-video

This commit is contained in:
openclaw committed 2026-10-19 14:36:20 +00:00
1 parent 3c5e5dd023
commit 8cd50b1c4d
16 files changed
+1553 -102

No files matched your search

+21
View File
@@ -0,0 +1,21 @@
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import type { ImageGenerationProvider, ImageGenerationRequest } from "openclaw/plugin-sdk/image-generation";
export interface SecretRefObject {
source: string;
provider: string;
id: string;
}
export interface GeminiImagePluginConfig {
baseUrl?: string;
apiKey?: string | SecretRefObject;
allowPrivateNetwork?: boolean;
defaultModel?: string;
models?: string[];
}
export declare function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined;
export declare function resolveGeminiImageConfig(cfg: ImageGenerationRequest["cfg"] | undefined, apiPluginConfig?: Record<string, unknown>): GeminiImagePluginConfig | undefined;
export declare function resolveConfiguredGeminiImageBaseUrl(pluginCfg?: GeminiImagePluginConfig): string;
export declare function buildGeminiImageGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }: {
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
getPluginConfig?: () => Record<string, unknown> | undefined;
}): ImageGenerationProvider;
+197
View File
@@ -0,0 +1,197 @@
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
const DEFAULT_GEMINI_IMAGE_BASE_URL = "http://host.incus.internal:8080/v1beta";
const DEFAULT_GEMINI_IMAGE_MODEL = "gemini-3.1-flash-image";
const DEFAULT_TIMEOUT_MS = 120_000;
export function resolvePluginApiKey(rawKey) {
if (typeof rawKey === "string") {
const trimmed = rawKey.trim();
const envMatch = trimmed.match(/^\$\{([A-Za-z0-9_]+)\}$/);
if (envMatch) {
return normalizeOptionalString(process.env[envMatch[1]]);
}
return normalizeOptionalString(trimmed);
}
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
return normalizeOptionalString(process.env[rawKey.id]);
}
return undefined;
}
export function resolveGeminiImageConfig(cfg, apiPluginConfig) {
const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["1h-tools"]?.config);
return toolsConfig?.geminiImage ?? toolsConfig?.image;
}
export function resolveConfiguredGeminiImageBaseUrl(pluginCfg) {
return (normalizeOptionalString(pluginCfg?.baseUrl) ??
normalizeOptionalString(process.env.GEMINI_IMAGE_BASE_URL) ??
DEFAULT_GEMINI_IMAGE_BASE_URL);
}
export function buildGeminiImageGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }) {
return {
id: "1h-gemini-image",
label: "1-h Gemini Image",
defaultModel: DEFAULT_GEMINI_IMAGE_MODEL,
models: [DEFAULT_GEMINI_IMAGE_MODEL],
isConfigured: (ctx) => {
const pluginCfg = resolveGeminiImageConfig(ctx.cfg, getPluginConfig?.());
if (resolvePluginApiKey(pluginCfg?.apiKey) ||
Boolean(pluginCfg?.apiKey) ||
normalizeOptionalString(process.env.CCLOAD_API_KEY) ||
normalizeOptionalString(process.env.GEMINI_API_KEY) ||
normalizeOptionalString(process.env.BIFROST_API_KEY)) {
return true;
}
return isProviderApiKeyConfigured({
provider: "google",
...ctx,
profileTypes: ["api_key"],
});
},
capabilities: {
generate: {
maxCount: 1,
supportsSize: false,
supportsAspectRatio: true,
supportsResolution: false,
},
edit: {
enabled: true,
maxCount: 1,
maxInputImages: 3,
supportsSize: false,
supportsAspectRatio: true,
supportsResolution: false,
},
geometry: {
aspectRatios: ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3", "21:9"],
},
output: {
formats: ["jpeg", "png"],
},
},
async generateImage(req) {
const pluginCfg = resolveGeminiImageConfig(req.cfg, getPluginConfig?.());
const explicitApiKey = resolvePluginApiKey(pluginCfg?.apiKey) ??
normalizeOptionalString(process.env.CCLOAD_API_KEY) ??
normalizeOptionalString(process.env.GEMINI_API_KEY) ??
normalizeOptionalString(process.env.BIFROST_API_KEY);
let apiKey = explicitApiKey;
if (!apiKey) {
const { resolveApiKeyForProvider } = await import("openclaw/plugin-sdk/provider-auth-runtime");
const auth = await resolveApiKeyForProvider({
provider: "google",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
});
if (!auth.apiKey) {
throw new Error("Gemini Image API key missing");
}
apiKey = auth.apiKey;
}
const [{ assertOkOrThrowHttpError, createProviderOperationDeadline, postJsonRequest, readProviderJsonResponse, resolveProviderHttpRequestConfig, resolveProviderOperationTimeoutMs, sanitizeConfiguredModelProviderRequest, },] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]);
const fetchFn = fetch;
const deadline = createProviderOperationDeadline({
timeoutMs: req.timeoutMs,
label: "Gemini Image Generation",
});
const rawBaseUrl = resolveConfiguredGeminiImageBaseUrl(pluginCfg);
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
const requestConfig = sanitizeConfiguredModelProviderRequest({
allowPrivateNetwork: allowPrivate,
});
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({
baseUrl: rawBaseUrl,
defaultBaseUrl: DEFAULT_GEMINI_IMAGE_BASE_URL,
allowPrivateNetwork: allowPrivate,
request: requestConfig,
defaultHeaders: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
},
provider: "google",
capability: "image",
transport: "http",
});
const model = normalizeOptionalString(req.model) ??
normalizeOptionalString(pluginCfg?.defaultModel) ??
DEFAULT_GEMINI_IMAGE_MODEL;
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
deadline,
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
});
const parts = [];
const inputImages = req.inputImages ?? [];
for (const img of inputImages) {
parts.push({
inlineData: {
mimeType: img.mimeType || "image/jpeg",
data: img.buffer.toString("base64"),
},
});
}
parts.push({
text: req.prompt,
});
const jsonBody = {
contents: [
{
parts,
},
],
generationConfig: {
responseMimeType: "image/jpeg",
},
};
const normalizedBaseUrl = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
const url = `${normalizedBaseUrl}/models/${model}:generateContent`;
const { response, release } = await postJsonRequest({
url,
headers,
body: jsonBody,
timeoutMs: submitTimeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
});
let payload;
try {
await assertOkOrThrowHttpError(response, "Gemini image generation request failed");
payload = await readProviderJsonResponse(response, "Gemini image generation failed");
}
finally {
await release();
}
if (payload.error?.message) {
throw new Error(`Gemini image generation error: ${payload.error.message}`);
}
const candidateParts = payload.candidates?.[0]?.content?.parts ?? [];
const generatedImages = [];
for (let i = 0; i < candidateParts.length; i++) {
const part = candidateParts[i];
if (part.inlineData?.data) {
const buffer = Buffer.from(part.inlineData.data, "base64");
const mimeType = part.inlineData.mimeType || "image/jpeg";
const ext = mimeType.includes("png") ? "png" : "jpg";
generatedImages.push({
buffer,
mimeType,
fileName: `image-${i + 1}.${ext}`,
});
}
}
if (generatedImages.length === 0) {
const textMessage = candidateParts.find((p) => p.text)?.text;
throw new Error(textMessage
? `Gemini returned text instead of an image: ${textMessage}`
: "Gemini image generation returned no image data");
}
return {
images: generatedImages,
model,
metadata: {
finishReason: payload.candidates?.[0]?.finishReason,
},
};
},
};
}
+27
View File
@@ -0,0 +1,27 @@
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import type { ImageGenerationProvider, ImageGenerationRequest } from "openclaw/plugin-sdk/image-generation";
export interface SecretRefObject {
source: string;
provider: string;
id: string;
}
export interface ImagePluginConfig {
baseUrl?: string;
apiKey?: string | SecretRefObject;
allowPrivateNetwork?: boolean;
defaultModel?: string;
models?: string[];
responseFormat?: "url" | "b64_json";
}
export declare function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined;
export declare function resolveImageConfig(cfg: ImageGenerationRequest["cfg"] | undefined, apiPluginConfig?: Record<string, unknown>): ImagePluginConfig | undefined;
export declare function resolveConfiguredImageBaseUrl(cfg: ImageGenerationRequest["cfg"], pluginCfg?: ImagePluginConfig): string;
export declare function resolveImageSize(params: {
size?: string;
aspectRatio?: string;
model?: string;
}): string;
export declare function buildOpenAIImageGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }: {
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
getPluginConfig?: () => Record<string, unknown> | undefined;
}): ImageGenerationProvider;
+305
View File
@@ -0,0 +1,305 @@
import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime";
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
const DEFAULT_OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-2";
const DEFAULT_TIMEOUT_MS = 120_000;
const DEFAULT_SIZE = "1024x1024";
const SUPPORTED_SIZES = [
"1024x1024",
"1536x1024",
"1024x1536",
"1792x1024",
"1024x1792",
"2048x2048",
"2048x1152",
"3840x2160",
];
const SUPPORTED_ASPECT_RATIOS = [
"1:1",
"16:9",
"9:16",
"4:3",
"3:4",
"3:2",
"2:3",
"21:9",
];
export function resolvePluginApiKey(rawKey) {
if (typeof rawKey === "string") {
const trimmed = rawKey.trim();
const envMatch = trimmed.match(/^\$\{([A-Za-z0-9_]+)\}$/);
if (envMatch) {
return normalizeOptionalString(process.env[envMatch[1]]);
}
return normalizeOptionalString(trimmed);
}
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
return normalizeOptionalString(process.env[rawKey.id]);
}
return undefined;
}
export function resolveImageConfig(cfg, apiPluginConfig) {
const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["1h-tools"]?.config);
return toolsConfig?.openaiImage ?? toolsConfig?.image;
}
export function resolveConfiguredImageBaseUrl(cfg, pluginCfg) {
return (normalizeOptionalString(pluginCfg?.baseUrl) ??
normalizeOptionalString(process.env.OPENAI_IMAGE_BASE_URL) ??
normalizeOptionalString(process.env.BIFROST_BASE_URL) ??
normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ??
normalizeOptionalString(process.env.OPENAI_BASE_URL) ??
DEFAULT_OPENAI_IMAGE_BASE_URL);
}
export function resolveImageSize(params) {
const explicitSize = normalizeOptionalString(params.size);
if (explicitSize) {
return explicitSize;
}
// Safe aspect ratio normalization preventing issue #168665 (proxy rejecting 2048x1152)
switch (normalizeOptionalString(params.aspectRatio)) {
case "1:1":
return "1024x1024";
case "16:9":
return "1792x1024";
case "9:16":
return "1024x1792";
case "4:3":
case "3:2":
return "1536x1024";
case "3:4":
case "2:3":
return "1024x1536";
case "21:9":
return "1792x1024";
default:
return DEFAULT_SIZE;
}
}
export function buildOpenAIImageGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }) {
return {
id: "1h-openai-image",
label: "1-h OpenAI Image",
defaultModel: DEFAULT_OPENAI_IMAGE_MODEL,
models: [DEFAULT_OPENAI_IMAGE_MODEL, "gpt-image-2.5", "gpt-image-2.5-flare", "gpt-image-2.5-sunburst"],
isConfigured: (ctx) => {
const pluginCfg = resolveImageConfig(ctx.cfg, getPluginConfig?.());
if (resolvePluginApiKey(pluginCfg?.apiKey) ||
Boolean(pluginCfg?.apiKey) ||
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ||
normalizeOptionalString(process.env.CCLOAD_API_KEY) ||
normalizeOptionalString(process.env.BIFROST_API_KEY) ||
normalizeOptionalString(process.env.OPENAI_API_KEY)) {
return true;
}
return isProviderApiKeyConfigured({
provider: "openai",
...ctx,
profileTypes: ["api_key"],
});
},
capabilities: {
generate: {
maxCount: 4,
supportsSize: true,
supportsAspectRatio: true,
supportsResolution: true,
},
edit: {
enabled: true,
maxCount: 4,
maxInputImages: 5,
supportsSize: true,
supportsAspectRatio: true,
supportsResolution: true,
},
geometry: {
sizes: [...SUPPORTED_SIZES],
aspectRatios: [...SUPPORTED_ASPECT_RATIOS],
resolutions: ["1K", "2K", "4K"],
},
output: {
formats: ["png", "jpeg", "webp"],
qualities: ["low", "medium", "high", "auto"],
},
},
async generateImage(req) {
const pluginCfg = resolveImageConfig(req.cfg, getPluginConfig?.());
const explicitApiKey = resolvePluginApiKey(pluginCfg?.apiKey) ??
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ??
normalizeOptionalString(process.env.CCLOAD_API_KEY) ??
normalizeOptionalString(process.env.BIFROST_API_KEY) ??
normalizeOptionalString(process.env.OPENAI_API_KEY);
let apiKey = explicitApiKey;
if (!apiKey) {
const { resolveApiKeyForProvider } = await import("openclaw/plugin-sdk/provider-auth-runtime");
const auth = await resolveApiKeyForProvider({
provider: "openai",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
modelApi: "openai-responses",
});
if (!auth.apiKey) {
throw new Error("OpenAI / Bifrost Image API key missing");
}
apiKey = auth.apiKey;
}
const [{ assertOkOrThrowHttpError, createProviderOperationDeadline, createProviderOperationTimeoutResolver, fetchWithTimeoutGuarded, postJsonRequest, postMultipartRequest, readProviderJsonResponse, resolveProviderHttpRequestConfig, resolveProviderOperationTimeoutMs, sanitizeConfiguredModelProviderRequest, },] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]);
const fetchFn = fetch;
const deadline = createProviderOperationDeadline({
timeoutMs: req.timeoutMs,
label: "OpenAI Image Generation",
});
const providerConfig = req.cfg?.models?.providers?.openai;
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
const requestConfig = sanitizeConfiguredModelProviderRequest({
...providerConfig?.request,
allowPrivateNetwork: allowPrivate,
});
const rawBaseUrl = resolveConfiguredImageBaseUrl(req.cfg, pluginCfg);
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({
baseUrl: rawBaseUrl,
defaultBaseUrl: DEFAULT_OPENAI_IMAGE_BASE_URL,
allowPrivateNetwork: allowPrivate,
request: requestConfig,
defaultHeaders: {
Authorization: `Bearer ${apiKey}`,
},
provider: "openai",
capability: "image",
transport: "http",
});
const model = normalizeOptionalString(req.model) ??
normalizeOptionalString(pluginCfg?.defaultModel) ??
DEFAULT_OPENAI_IMAGE_MODEL;
const size = resolveImageSize({
size: req.size,
aspectRatio: req.aspectRatio,
model,
});
const count = Math.max(1, Math.min(req.count ?? 1, 4));
const inputImages = req.inputImages ?? [];
const isEdit = inputImages.length > 0;
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
deadline,
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
});
let response;
let release;
const responseFormat = pluginCfg?.responseFormat ?? "url";
if (isEdit) {
const form = new FormData();
form.set("model", model);
form.set("prompt", req.prompt);
form.set("n", String(count));
form.set("size", size);
form.set("response_format", responseFormat);
inputImages.forEach((img, idx) => {
const mime = img.mimeType || "image/png";
const ext = extensionForMime(mime)?.slice(1) || "png";
const name = img.fileName || `input-${idx + 1}.${ext}`;
form.append(idx === 0 ? "image" : "image[]", new File([bufferToBlobPart(img.buffer)], name, { type: mime }));
});
const multipartHeaders = new Headers(headers);
multipartHeaders.delete("Content-Type");
({ response, release } = await postMultipartRequest({
url: `${baseUrl}/images/edits`,
headers: multipartHeaders,
body: form,
timeoutMs: submitTimeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
}));
}
else {
const jsonHeaders = new Headers(headers);
jsonHeaders.set("Content-Type", "application/json");
const jsonBody = {
model,
prompt: req.prompt,
n: count,
size,
response_format: responseFormat,
};
if (req.quality) {
jsonBody.quality = req.quality;
}
({ response, release } = await postJsonRequest({
url: `${baseUrl}/images/generations`,
headers: jsonHeaders,
body: jsonBody,
timeoutMs: submitTimeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
}));
}
let payload;
try {
await assertOkOrThrowHttpError(response, "Image generation request failed");
payload = await readProviderJsonResponse(response, "OpenAI image generation failed");
}
finally {
await release();
}
if (payload.error?.message) {
throw new Error(`Image generation error: ${payload.error.message}`);
}
const dataItems = payload.data ?? [];
if (dataItems.length === 0) {
throw new Error("Image generation returned no image data");
}
const generatedImages = [];
const downloadTimeout = createProviderOperationTimeoutResolver({
deadline,
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
});
for (let i = 0; i < dataItems.length; i++) {
const item = dataItems[i];
if (item.b64_json) {
const buffer = Buffer.from(item.b64_json, "base64");
generatedImages.push({
buffer,
mimeType: "image/png",
fileName: `image-${i + 1}.png`,
revisedPrompt: item.revised_prompt,
});
}
else if (item.url) {
const dlResult = await fetchWithTimeoutGuarded(item.url, { method: "GET" }, downloadTimeout(), fetchFn, {
allowPrivateNetwork,
auditContext: "openai-image-download",
});
try {
await assertOkOrThrowHttpError(dlResult.response, "Failed to download generated image");
const arrayBuf = await dlResult.response.arrayBuffer();
const mime = dlResult.response.headers.get("content-type") || "image/png";
const ext = extensionForMime(mime)?.slice(1) || "png";
generatedImages.push({
buffer: Buffer.from(arrayBuf),
mimeType: mime,
fileName: `image-${i + 1}.${ext}`,
revisedPrompt: item.revised_prompt,
});
}
finally {
await dlResult.release();
}
}
}
if (generatedImages.length === 0) {
throw new Error("Failed to extract any valid image from provider response");
}
return {
images: generatedImages,
model,
metadata: {
created: payload.created,
size,
},
};
},
};
}