303 lines
9.0 KiB
TypeScript
303 lines
9.0 KiB
TypeScript
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
|
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
import type {
|
|
GeneratedImageAsset,
|
|
ImageGenerationProvider,
|
|
ImageGenerationRequest,
|
|
ImageGenerationResult,
|
|
} from "openclaw/plugin-sdk/image-generation";
|
|
|
|
const DEFAULT_GEMINI_IMAGE_BASE_URL = "http://host.incus.internal:8080/v1beta";
|
|
const DEFAULT_GEMINI_IMAGE_MODEL = "gemini-3.1-flash-image";
|
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
|
|
export interface SecretRefObject {
|
|
source: string;
|
|
provider: string;
|
|
id: string;
|
|
}
|
|
|
|
export interface GeminiImagePluginConfig {
|
|
baseUrl?: string;
|
|
apiKey?: string | SecretRefObject;
|
|
allowPrivateNetwork?: boolean;
|
|
defaultModel?: string;
|
|
models?: string[];
|
|
}
|
|
|
|
export function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined {
|
|
if (typeof rawKey === "string") {
|
|
const trimmed = rawKey.trim();
|
|
const envMatch = trimmed.match(/^\$\{([A-Za-z0-9_]+)\}$/);
|
|
if (envMatch) {
|
|
return normalizeOptionalString(process.env[envMatch[1]]);
|
|
}
|
|
return normalizeOptionalString(trimmed);
|
|
}
|
|
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
|
|
return normalizeOptionalString(process.env[rawKey.id]);
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
export function resolveGeminiImageConfig(
|
|
cfg: ImageGenerationRequest["cfg"] | undefined,
|
|
apiPluginConfig?: Record<string, unknown>,
|
|
): GeminiImagePluginConfig | undefined {
|
|
const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["1h-tools"]?.config) as
|
|
| {
|
|
geminiImage?: GeminiImagePluginConfig;
|
|
image?: GeminiImagePluginConfig;
|
|
}
|
|
| undefined;
|
|
|
|
return toolsConfig?.geminiImage ?? toolsConfig?.image;
|
|
}
|
|
|
|
export function resolveConfiguredGeminiImageBaseUrl(
|
|
pluginCfg?: GeminiImagePluginConfig,
|
|
): string {
|
|
return (
|
|
normalizeOptionalString(pluginCfg?.baseUrl) ??
|
|
normalizeOptionalString(process.env.GEMINI_IMAGE_BASE_URL) ??
|
|
DEFAULT_GEMINI_IMAGE_BASE_URL
|
|
);
|
|
}
|
|
|
|
export function buildGeminiImageGenerationProvider({
|
|
isProviderApiKeyConfigured,
|
|
getPluginConfig,
|
|
}: {
|
|
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
|
|
getPluginConfig?: () => Record<string, unknown> | undefined;
|
|
}): ImageGenerationProvider {
|
|
return {
|
|
id: "1h-gemini-image",
|
|
label: "1-h Gemini Image",
|
|
defaultModel: DEFAULT_GEMINI_IMAGE_MODEL,
|
|
models: [DEFAULT_GEMINI_IMAGE_MODEL],
|
|
isConfigured: (ctx) => {
|
|
const pluginCfg = resolveGeminiImageConfig(ctx.cfg, getPluginConfig?.());
|
|
if (
|
|
resolvePluginApiKey(pluginCfg?.apiKey) ||
|
|
Boolean(pluginCfg?.apiKey) ||
|
|
normalizeOptionalString(process.env.CCLOAD_API_KEY) ||
|
|
normalizeOptionalString(process.env.GEMINI_API_KEY) ||
|
|
normalizeOptionalString(process.env.BIFROST_API_KEY)
|
|
) {
|
|
return true;
|
|
}
|
|
return isProviderApiKeyConfigured({
|
|
provider: "google",
|
|
...ctx,
|
|
profileTypes: ["api_key"],
|
|
});
|
|
},
|
|
capabilities: {
|
|
generate: {
|
|
maxCount: 1,
|
|
supportsSize: false,
|
|
supportsAspectRatio: true,
|
|
supportsResolution: false,
|
|
},
|
|
edit: {
|
|
enabled: true,
|
|
maxCount: 1,
|
|
maxInputImages: 3,
|
|
supportsSize: false,
|
|
supportsAspectRatio: true,
|
|
supportsResolution: false,
|
|
},
|
|
geometry: {
|
|
aspectRatios: ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3", "21:9"],
|
|
},
|
|
output: {
|
|
formats: ["jpeg", "png"],
|
|
},
|
|
},
|
|
async generateImage(req: ImageGenerationRequest): Promise<ImageGenerationResult> {
|
|
const pluginCfg = resolveGeminiImageConfig(req.cfg, getPluginConfig?.());
|
|
const explicitApiKey =
|
|
resolvePluginApiKey(pluginCfg?.apiKey) ??
|
|
normalizeOptionalString(process.env.CCLOAD_API_KEY) ??
|
|
normalizeOptionalString(process.env.GEMINI_API_KEY) ??
|
|
normalizeOptionalString(process.env.BIFROST_API_KEY);
|
|
|
|
let apiKey = explicitApiKey;
|
|
if (!apiKey) {
|
|
const { resolveApiKeyForProvider } = await import(
|
|
"openclaw/plugin-sdk/provider-auth-runtime"
|
|
);
|
|
const auth = await resolveApiKeyForProvider({
|
|
provider: "google",
|
|
cfg: req.cfg,
|
|
agentDir: req.agentDir,
|
|
store: req.authStore,
|
|
});
|
|
if (!auth.apiKey) {
|
|
throw new Error("Gemini Image API key missing");
|
|
}
|
|
apiKey = auth.apiKey;
|
|
}
|
|
|
|
const [
|
|
{
|
|
assertOkOrThrowHttpError,
|
|
createProviderOperationDeadline,
|
|
postJsonRequest,
|
|
readProviderJsonResponse,
|
|
resolveProviderHttpRequestConfig,
|
|
resolveProviderOperationTimeoutMs,
|
|
sanitizeConfiguredModelProviderRequest,
|
|
},
|
|
] = await Promise.all([import("openclaw/plugin-sdk/provider-http")]);
|
|
|
|
const fetchFn = fetch;
|
|
const deadline = createProviderOperationDeadline({
|
|
timeoutMs: req.timeoutMs,
|
|
label: "Gemini Image Generation",
|
|
});
|
|
|
|
const rawBaseUrl = resolveConfiguredGeminiImageBaseUrl(pluginCfg);
|
|
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
|
|
const requestConfig = sanitizeConfiguredModelProviderRequest({
|
|
allowPrivateNetwork: allowPrivate,
|
|
});
|
|
|
|
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
|
resolveProviderHttpRequestConfig({
|
|
baseUrl: rawBaseUrl,
|
|
defaultBaseUrl: DEFAULT_GEMINI_IMAGE_BASE_URL,
|
|
allowPrivateNetwork: allowPrivate,
|
|
request: requestConfig,
|
|
defaultHeaders: {
|
|
Authorization: `Bearer ${apiKey}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
provider: "google",
|
|
capability: "image",
|
|
transport: "http",
|
|
});
|
|
|
|
const model =
|
|
normalizeOptionalString(req.model) ??
|
|
normalizeOptionalString(pluginCfg?.defaultModel) ??
|
|
DEFAULT_GEMINI_IMAGE_MODEL;
|
|
|
|
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
|
|
deadline,
|
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
|
});
|
|
|
|
const parts: Array<Record<string, unknown>> = [];
|
|
const inputImages = req.inputImages ?? [];
|
|
|
|
for (const img of inputImages) {
|
|
parts.push({
|
|
inlineData: {
|
|
mimeType: img.mimeType || "image/jpeg",
|
|
data: img.buffer.toString("base64"),
|
|
},
|
|
});
|
|
}
|
|
|
|
parts.push({
|
|
text: req.prompt,
|
|
});
|
|
|
|
const jsonBody: Record<string, unknown> = {
|
|
contents: [
|
|
{
|
|
parts,
|
|
},
|
|
],
|
|
generationConfig: {
|
|
responseMimeType: "image/jpeg",
|
|
},
|
|
};
|
|
|
|
const normalizedBaseUrl = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
const url = `${normalizedBaseUrl}/models/${model}:generateContent`;
|
|
|
|
const { response, release } = await postJsonRequest({
|
|
url,
|
|
headers,
|
|
body: jsonBody,
|
|
timeoutMs: submitTimeoutMs,
|
|
fetchFn,
|
|
allowPrivateNetwork,
|
|
dispatcherPolicy,
|
|
});
|
|
|
|
interface GeminiGenerateContentResponse {
|
|
candidates?: Array<{
|
|
content?: {
|
|
parts?: Array<{
|
|
text?: string;
|
|
inlineData?: {
|
|
mimeType?: string;
|
|
data?: string;
|
|
};
|
|
}>;
|
|
};
|
|
finishReason?: string;
|
|
}>;
|
|
error?: {
|
|
code?: number;
|
|
message?: string;
|
|
status?: string;
|
|
};
|
|
}
|
|
|
|
let payload: GeminiGenerateContentResponse;
|
|
try {
|
|
await assertOkOrThrowHttpError(response, "Gemini image generation request failed");
|
|
payload = await readProviderJsonResponse<GeminiGenerateContentResponse>(
|
|
response,
|
|
"Gemini image generation failed",
|
|
);
|
|
} finally {
|
|
await release();
|
|
}
|
|
|
|
if (payload.error?.message) {
|
|
throw new Error(`Gemini image generation error: ${payload.error.message}`);
|
|
}
|
|
|
|
const candidateParts = payload.candidates?.[0]?.content?.parts ?? [];
|
|
const generatedImages: GeneratedImageAsset[] = [];
|
|
|
|
for (let i = 0; i < candidateParts.length; i++) {
|
|
const part = candidateParts[i];
|
|
if (part.inlineData?.data) {
|
|
const buffer = Buffer.from(part.inlineData.data, "base64");
|
|
const mimeType = part.inlineData.mimeType || "image/jpeg";
|
|
const ext = mimeType.includes("png") ? "png" : "jpg";
|
|
generatedImages.push({
|
|
buffer,
|
|
mimeType,
|
|
fileName: `image-${i + 1}.${ext}`,
|
|
});
|
|
}
|
|
}
|
|
|
|
if (generatedImages.length === 0) {
|
|
const textMessage = candidateParts.find((p) => p.text)?.text;
|
|
throw new Error(
|
|
textMessage
|
|
? `Gemini returned text instead of an image: ${textMessage}`
|
|
: "Gemini image generation returned no image data",
|
|
);
|
|
}
|
|
|
|
return {
|
|
images: generatedImages,
|
|
model,
|
|
metadata: {
|
|
finishReason: payload.candidates?.[0]?.finishReason,
|
|
},
|
|
};
|
|
},
|
|
};
|
|
}
|