420 lines
13 KiB
TypeScript
420 lines
13 KiB
TypeScript
import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime";
|
|
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
|
|
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
|
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
import type {
|
|
GeneratedImageAsset,
|
|
ImageGenerationProvider,
|
|
ImageGenerationRequest,
|
|
ImageGenerationResult,
|
|
} from "openclaw/plugin-sdk/image-generation";
|
|
|
|
const DEFAULT_OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
|
|
const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-2";
|
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
const DEFAULT_SIZE = "1024x1024";
|
|
|
|
const SUPPORTED_SIZES = [
|
|
"1024x1024",
|
|
"1536x1024",
|
|
"1024x1536",
|
|
"1792x1024",
|
|
"1024x1792",
|
|
"2048x2048",
|
|
"2048x1152",
|
|
"3840x2160",
|
|
] as const;
|
|
|
|
const SUPPORTED_ASPECT_RATIOS = [
|
|
"1:1",
|
|
"16:9",
|
|
"9:16",
|
|
"4:3",
|
|
"3:4",
|
|
"3:2",
|
|
"2:3",
|
|
"21:9",
|
|
] as const;
|
|
|
|
export interface SecretRefObject {
|
|
source: string;
|
|
provider: string;
|
|
id: string;
|
|
}
|
|
|
|
export interface ImagePluginConfig {
|
|
baseUrl?: string;
|
|
apiKey?: string | SecretRefObject;
|
|
allowPrivateNetwork?: boolean;
|
|
defaultModel?: string;
|
|
models?: string[];
|
|
}
|
|
|
|
export function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined {
|
|
if (typeof rawKey === "string") {
|
|
const trimmed = rawKey.trim();
|
|
const envMatch = trimmed.match(/^\$\{([A-Za-z0-9_]+)\}$/);
|
|
if (envMatch) {
|
|
return normalizeOptionalString(process.env[envMatch[1]]);
|
|
}
|
|
return normalizeOptionalString(trimmed);
|
|
}
|
|
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
|
|
return normalizeOptionalString(process.env[rawKey.id]);
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
export function resolveImageConfig(
|
|
cfg: ImageGenerationRequest["cfg"] | undefined,
|
|
apiPluginConfig?: Record<string, unknown>,
|
|
): ImagePluginConfig | undefined {
|
|
const direct = apiPluginConfig?.image as ImagePluginConfig | undefined;
|
|
const fromCfg = cfg?.plugins?.entries?.["1h-tools"]?.config?.image as ImagePluginConfig | undefined;
|
|
return direct ?? fromCfg;
|
|
}
|
|
|
|
export function resolveConfiguredImageBaseUrl(
|
|
cfg: ImageGenerationRequest["cfg"],
|
|
pluginCfg?: ImagePluginConfig,
|
|
): string {
|
|
return (
|
|
normalizeOptionalString(pluginCfg?.baseUrl) ??
|
|
normalizeOptionalString(process.env.OPENAI_IMAGE_BASE_URL) ??
|
|
normalizeOptionalString(process.env.BIFROST_BASE_URL) ??
|
|
normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ??
|
|
normalizeOptionalString(process.env.OPENAI_BASE_URL) ??
|
|
DEFAULT_OPENAI_IMAGE_BASE_URL
|
|
);
|
|
}
|
|
|
|
export function resolveImageSize(params: {
|
|
size?: string;
|
|
aspectRatio?: string;
|
|
model?: string;
|
|
}): string {
|
|
const explicitSize = normalizeOptionalString(params.size);
|
|
if (explicitSize) {
|
|
return explicitSize;
|
|
}
|
|
|
|
// Safe aspect ratio normalization preventing issue #168665 (proxy rejecting 2048x1152)
|
|
switch (normalizeOptionalString(params.aspectRatio)) {
|
|
case "1:1":
|
|
return "1024x1024";
|
|
case "16:9":
|
|
return "1792x1024";
|
|
case "9:16":
|
|
return "1024x1792";
|
|
case "4:3":
|
|
case "3:2":
|
|
return "1536x1024";
|
|
case "3:4":
|
|
case "2:3":
|
|
return "1024x1536";
|
|
case "21:9":
|
|
return "1792x1024";
|
|
default:
|
|
return DEFAULT_SIZE;
|
|
}
|
|
}
|
|
|
|
export function buildOpenAIImageGenerationProvider({
|
|
isProviderApiKeyConfigured,
|
|
getPluginConfig,
|
|
}: {
|
|
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
|
|
getPluginConfig?: () => Record<string, unknown> | undefined;
|
|
}): ImageGenerationProvider {
|
|
return {
|
|
id: "1h-image",
|
|
label: "1-h Image",
|
|
defaultModel: DEFAULT_OPENAI_IMAGE_MODEL,
|
|
models: [DEFAULT_OPENAI_IMAGE_MODEL, "gpt-image-1.5", "gemini-3.1-flash-image"],
|
|
isConfigured: (ctx) => {
|
|
const pluginCfg = resolveImageConfig(ctx.cfg, getPluginConfig?.());
|
|
if (
|
|
resolvePluginApiKey(pluginCfg?.apiKey) ||
|
|
Boolean(pluginCfg?.apiKey) ||
|
|
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ||
|
|
normalizeOptionalString(process.env.CCLOAD_API_KEY) ||
|
|
normalizeOptionalString(process.env.BIFROST_API_KEY) ||
|
|
normalizeOptionalString(process.env.OPENAI_API_KEY)
|
|
) {
|
|
return true;
|
|
}
|
|
return isProviderApiKeyConfigured({
|
|
provider: "openai",
|
|
...ctx,
|
|
profileTypes: ["api_key"],
|
|
});
|
|
},
|
|
capabilities: {
|
|
generate: {
|
|
maxCount: 4,
|
|
supportsSize: true,
|
|
supportsAspectRatio: true,
|
|
supportsResolution: true,
|
|
},
|
|
edit: {
|
|
enabled: true,
|
|
maxCount: 4,
|
|
maxInputImages: 5,
|
|
supportsSize: true,
|
|
supportsAspectRatio: true,
|
|
supportsResolution: true,
|
|
},
|
|
geometry: {
|
|
sizes: [...SUPPORTED_SIZES],
|
|
aspectRatios: [...SUPPORTED_ASPECT_RATIOS],
|
|
resolutions: ["1K", "2K", "4K"],
|
|
},
|
|
output: {
|
|
formats: ["png", "jpeg", "webp"],
|
|
qualities: ["low", "medium", "high", "auto"],
|
|
},
|
|
},
|
|
async generateImage(req: ImageGenerationRequest): Promise<ImageGenerationResult> {
|
|
const pluginCfg = resolveImageConfig(req.cfg, getPluginConfig?.());
|
|
const explicitApiKey =
|
|
resolvePluginApiKey(pluginCfg?.apiKey) ??
|
|
normalizeOptionalString(process.env.OPENAI_IMAGE_API_KEY) ??
|
|
normalizeOptionalString(process.env.CCLOAD_API_KEY) ??
|
|
normalizeOptionalString(process.env.BIFROST_API_KEY) ??
|
|
normalizeOptionalString(process.env.OPENAI_API_KEY);
|
|
|
|
let apiKey = explicitApiKey;
|
|
if (!apiKey) {
|
|
const { resolveApiKeyForProvider } = await import(
|
|
"openclaw/plugin-sdk/provider-auth-runtime"
|
|
);
|
|
const auth = await resolveApiKeyForProvider({
|
|
provider: "openai",
|
|
cfg: req.cfg,
|
|
agentDir: req.agentDir,
|
|
store: req.authStore,
|
|
modelApi: "openai-responses",
|
|
});
|
|
if (!auth.apiKey) {
|
|
throw new Error("OpenAI / Bifrost Image API key missing");
|
|
}
|
|
apiKey = auth.apiKey;
|
|
}
|
|
|
|
const [
|
|
{
|
|
assertOkOrThrowHttpError,
|
|
createProviderOperationDeadline,
|
|
createProviderOperationTimeoutResolver,
|
|
fetchWithTimeoutGuarded,
|
|
postJsonRequest,
|
|
postMultipartRequest,
|
|
readProviderJsonResponse,
|
|
resolveProviderHttpRequestConfig,
|
|
resolveProviderOperationTimeoutMs,
|
|
sanitizeConfiguredModelProviderRequest,
|
|
},
|
|
] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]);
|
|
|
|
const fetchFn = fetch;
|
|
const deadline = createProviderOperationDeadline({
|
|
timeoutMs: req.timeoutMs,
|
|
label: "OpenAI Image Generation",
|
|
});
|
|
|
|
const providerConfig = req.cfg?.models?.providers?.openai;
|
|
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
|
|
const requestConfig = sanitizeConfiguredModelProviderRequest({
|
|
...providerConfig?.request,
|
|
allowPrivateNetwork: allowPrivate,
|
|
});
|
|
|
|
const rawBaseUrl = resolveConfiguredImageBaseUrl(req.cfg, pluginCfg);
|
|
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
|
resolveProviderHttpRequestConfig({
|
|
baseUrl: rawBaseUrl,
|
|
defaultBaseUrl: DEFAULT_OPENAI_IMAGE_BASE_URL,
|
|
allowPrivateNetwork: allowPrivate,
|
|
request: requestConfig,
|
|
defaultHeaders: {
|
|
Authorization: `Bearer ${apiKey}`,
|
|
},
|
|
provider: "openai",
|
|
capability: "image",
|
|
transport: "http",
|
|
});
|
|
|
|
const model =
|
|
normalizeOptionalString(req.model) ??
|
|
normalizeOptionalString(pluginCfg?.defaultModel) ??
|
|
DEFAULT_OPENAI_IMAGE_MODEL;
|
|
|
|
const size = resolveImageSize({
|
|
size: req.size,
|
|
aspectRatio: req.aspectRatio,
|
|
model,
|
|
});
|
|
|
|
const count = Math.max(1, Math.min(req.count ?? 1, 4));
|
|
const inputImages = req.inputImages ?? [];
|
|
const isEdit = inputImages.length > 0;
|
|
|
|
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
|
|
deadline,
|
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
|
});
|
|
|
|
let response: Response;
|
|
let release: () => Promise<void>;
|
|
|
|
if (isEdit) {
|
|
const form = new FormData();
|
|
form.set("model", model);
|
|
form.set("prompt", req.prompt);
|
|
form.set("n", String(count));
|
|
form.set("size", size);
|
|
form.set("response_format", "b64_json");
|
|
|
|
inputImages.forEach((img, idx) => {
|
|
const mime = img.mimeType || "image/png";
|
|
const ext = extensionForMime(mime)?.slice(1) || "png";
|
|
const name = img.fileName || `input-${idx + 1}.${ext}`;
|
|
form.append(
|
|
idx === 0 ? "image" : "image[]",
|
|
new File([bufferToBlobPart(img.buffer)], name, { type: mime }),
|
|
);
|
|
});
|
|
|
|
const multipartHeaders = new Headers(headers);
|
|
multipartHeaders.delete("Content-Type");
|
|
|
|
({ response, release } = await postMultipartRequest({
|
|
url: `${baseUrl}/images/edits`,
|
|
headers: multipartHeaders,
|
|
body: form,
|
|
timeoutMs: submitTimeoutMs,
|
|
fetchFn,
|
|
allowPrivateNetwork,
|
|
dispatcherPolicy,
|
|
}));
|
|
} else {
|
|
const jsonHeaders = new Headers(headers);
|
|
jsonHeaders.set("Content-Type", "application/json");
|
|
|
|
const jsonBody: Record<string, unknown> = {
|
|
model,
|
|
prompt: req.prompt,
|
|
n: count,
|
|
size,
|
|
response_format: "b64_json",
|
|
};
|
|
|
|
if (req.quality) {
|
|
jsonBody.quality = req.quality;
|
|
}
|
|
|
|
({ response, release } = await postJsonRequest({
|
|
url: `${baseUrl}/images/generations`,
|
|
headers: jsonHeaders,
|
|
body: jsonBody,
|
|
timeoutMs: submitTimeoutMs,
|
|
fetchFn,
|
|
allowPrivateNetwork,
|
|
dispatcherPolicy,
|
|
}));
|
|
}
|
|
|
|
interface OpenAIImageApiResponse {
|
|
created?: number;
|
|
data?: Array<{
|
|
b64_json?: string;
|
|
url?: string;
|
|
revised_prompt?: string;
|
|
}>;
|
|
error?: {
|
|
message?: string;
|
|
code?: string;
|
|
type?: string;
|
|
};
|
|
}
|
|
|
|
let payload: OpenAIImageApiResponse;
|
|
try {
|
|
await assertOkOrThrowHttpError(response, "Image generation request failed");
|
|
payload = await readProviderJsonResponse<OpenAIImageApiResponse>(
|
|
response,
|
|
"OpenAI image generation failed",
|
|
);
|
|
} finally {
|
|
await release();
|
|
}
|
|
|
|
if (payload.error?.message) {
|
|
throw new Error(`Image generation error: ${payload.error.message}`);
|
|
}
|
|
|
|
const dataItems = payload.data ?? [];
|
|
if (dataItems.length === 0) {
|
|
throw new Error("Image generation returned no image data");
|
|
}
|
|
|
|
const generatedImages: GeneratedImageAsset[] = [];
|
|
const downloadTimeout = createProviderOperationTimeoutResolver({
|
|
deadline,
|
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
|
});
|
|
|
|
for (let i = 0; i < dataItems.length; i++) {
|
|
const item = dataItems[i];
|
|
if (item.b64_json) {
|
|
const buffer = Buffer.from(item.b64_json, "base64");
|
|
generatedImages.push({
|
|
buffer,
|
|
mimeType: "image/png",
|
|
fileName: `image-${i + 1}.png`,
|
|
revisedPrompt: item.revised_prompt,
|
|
});
|
|
} else if (item.url) {
|
|
const dlResult = await fetchWithTimeoutGuarded(
|
|
item.url,
|
|
{ method: "GET" },
|
|
downloadTimeout(),
|
|
fetchFn,
|
|
{
|
|
allowPrivateNetwork,
|
|
auditContext: "openai-image-download",
|
|
},
|
|
);
|
|
try {
|
|
await assertOkOrThrowHttpError(dlResult.response, "Failed to download generated image");
|
|
const arrayBuf = await dlResult.response.arrayBuffer();
|
|
const mime = dlResult.response.headers.get("content-type") || "image/png";
|
|
const ext = extensionForMime(mime)?.slice(1) || "png";
|
|
generatedImages.push({
|
|
buffer: Buffer.from(arrayBuf),
|
|
mimeType: mime,
|
|
fileName: `image-${i + 1}.${ext}`,
|
|
revisedPrompt: item.revised_prompt,
|
|
});
|
|
} finally {
|
|
await dlResult.release();
|
|
}
|
|
}
|
|
}
|
|
|
|
if (generatedImages.length === 0) {
|
|
throw new Error("Failed to extract any valid image from provider response");
|
|
}
|
|
|
|
return {
|
|
images: generatedImages,
|
|
model,
|
|
metadata: {
|
|
created: payload.created,
|
|
size,
|
|
},
|
|
};
|
|
},
|
|
};
|
|
}
|