feat: migrate openai-video provider into tools plugin
This commit is contained in:
1 parent
42ef37f10a
commit
720f48844d
13 files changed
+1306
-38
No files matched your search
Vendored
+14
-2
@@ -1,2 +1,14 @@
|
|||||||
declare const _default: import("openclaw/plugin-sdk/tool-plugin").DefinedToolPluginEntry;
|
declare const entry: Omit<{
|
||||||
export default _default;
|
id: string;
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
kind?: import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["kind"];
|
||||||
|
configSchema?: import("openclaw/plugin-sdk/core").OpenClawPluginConfigSchema | (() => import("openclaw/plugin-sdk/core").OpenClawPluginConfigSchema);
|
||||||
|
reload?: import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["reload"];
|
||||||
|
nodeHostCommands?: import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["nodeHostCommands"];
|
||||||
|
securityAuditCollectors?: import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["securityAuditCollectors"];
|
||||||
|
register: NonNullable<import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["register"]>;
|
||||||
|
}, "configSchema"> & {
|
||||||
|
configSchema: import("openclaw/plugin-sdk/core").OpenClawPluginConfigSchema;
|
||||||
|
};
|
||||||
|
export default entry;
|
||||||
Vendored
+33
-14
@@ -1,17 +1,36 @@
|
|||||||
import { Type } from "typebox";
|
import { jsonResult } from "openclaw/plugin-sdk/core";
|
||||||
import { defineToolPlugin } from "openclaw/plugin-sdk/tool-plugin";
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||||
export default defineToolPlugin({
|
import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin";
|
||||||
|
import { echoTool } from "./tools/echo.js";
|
||||||
|
import { buildOpenAIVideoGenerationProvider } from "./video/index.js";
|
||||||
|
const entry = definePluginEntry({
|
||||||
id: "tools",
|
id: "tools",
|
||||||
name: "Tools",
|
name: "Tools",
|
||||||
description: "Add Tools tools to OpenClaw.",
|
description: "Custom Tools and Media Providers for OpenClaw.",
|
||||||
tools: (tool) => [
|
register(api) {
|
||||||
tool({
|
api.registerTool({
|
||||||
name: "echo",
|
name: echoTool.name,
|
||||||
description: "Echo input text.",
|
label: echoTool.label,
|
||||||
parameters: Type.Object({
|
description: echoTool.description,
|
||||||
input: Type.String({ description: "Text to echo." }),
|
parameters: echoTool.parameters,
|
||||||
}),
|
execute: async (_toolCallId, params) => {
|
||||||
execute: async ({ input }) => ({ input }),
|
const result = await echoTool.execute(params);
|
||||||
}),
|
return jsonResult(result);
|
||||||
],
|
},
|
||||||
|
});
|
||||||
|
api.registerVideoGenerationProvider(buildOpenAIVideoGenerationProvider({
|
||||||
|
isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured,
|
||||||
|
getPluginConfig: () => api.pluginConfig,
|
||||||
|
}));
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
Object.defineProperty(entry, toolPluginMetadataSymbol, {
|
||||||
|
value: {
|
||||||
|
id: "tools",
|
||||||
|
name: "Tools",
|
||||||
|
description: "Custom Tools and Media Providers for OpenClaw.",
|
||||||
|
tools: [echoTool],
|
||||||
|
},
|
||||||
|
enumerable: false,
|
||||||
|
});
|
||||||
|
export default entry;
|
||||||
Vendored
+16
@@ -0,0 +1,16 @@
|
|||||||
|
import { Type, type Static } from "typebox";
|
||||||
|
export declare const EchoInputSchema: Type.TObject<{
|
||||||
|
input: Type.TString;
|
||||||
|
}>;
|
||||||
|
export type EchoInput = Static<typeof EchoInputSchema>;
|
||||||
|
export declare const echoTool: {
|
||||||
|
name: string;
|
||||||
|
label: string;
|
||||||
|
description: string;
|
||||||
|
parameters: Type.TObject<{
|
||||||
|
input: Type.TString;
|
||||||
|
}>;
|
||||||
|
execute: ({ input }: EchoInput) => Promise<{
|
||||||
|
input: string;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
Vendored
+11
@@ -0,0 +1,11 @@
|
|||||||
|
import { Type } from "typebox";
|
||||||
|
export const EchoInputSchema = Type.Object({
|
||||||
|
input: Type.String({ description: "Text to echo." }),
|
||||||
|
});
|
||||||
|
export const echoTool = {
|
||||||
|
name: "echo",
|
||||||
|
label: "Echo",
|
||||||
|
description: "Echo input text.",
|
||||||
|
parameters: EchoInputSchema,
|
||||||
|
execute: async ({ input }) => ({ input }),
|
||||||
|
};
|
||||||
Vendored
+60
@@ -0,0 +1,60 @@
|
|||||||
|
import { type MediaKind } from "openclaw/plugin-sdk/media-mime";
|
||||||
|
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||||
|
import type { createProviderOperationDeadline, postMultipartRequest } from "openclaw/plugin-sdk/provider-http";
|
||||||
|
import type { VideoGenerationProvider, VideoGenerationRequest } from "openclaw/plugin-sdk/video-generation";
|
||||||
|
declare const OPENAI_VIDEO_SIZES: readonly ["720x1280", "1280x720", "1024x1792", "1792x1024"];
|
||||||
|
export type OpenAIVideoRequestPolicy = {
|
||||||
|
allowPrivateNetwork: boolean;
|
||||||
|
dispatcherPolicy?: Parameters<typeof postMultipartRequest>[0]["dispatcherPolicy"];
|
||||||
|
};
|
||||||
|
export type OpenAIVideoStatus = "queued" | "in_progress" | "completed" | "failed";
|
||||||
|
export type OpenAIReferenceAsset = {
|
||||||
|
kind: Extract<MediaKind, "image" | "video">;
|
||||||
|
file: File;
|
||||||
|
};
|
||||||
|
export type OpenAIVideoResponse = {
|
||||||
|
id?: string;
|
||||||
|
model?: string;
|
||||||
|
status?: OpenAIVideoStatus;
|
||||||
|
prompt?: string | null;
|
||||||
|
seconds?: string;
|
||||||
|
size?: string;
|
||||||
|
error?: string | {
|
||||||
|
code?: string;
|
||||||
|
message?: string;
|
||||||
|
} | null;
|
||||||
|
};
|
||||||
|
export interface SecretRefObject {
|
||||||
|
source: string;
|
||||||
|
provider: string;
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
export interface VideoPluginConfig {
|
||||||
|
baseUrl?: string;
|
||||||
|
apiKey?: string | SecretRefObject;
|
||||||
|
allowPrivateNetwork?: boolean;
|
||||||
|
defaultModel?: string;
|
||||||
|
models?: string[];
|
||||||
|
}
|
||||||
|
export declare function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined;
|
||||||
|
export declare function resolveVideoConfig(cfg: VideoGenerationRequest["cfg"] | undefined, apiPluginConfig?: Record<string, unknown>): VideoPluginConfig | undefined;
|
||||||
|
export declare function resolveConfiguredOpenAIBaseUrl(cfg: VideoGenerationRequest["cfg"], pluginCfg?: VideoPluginConfig): string;
|
||||||
|
export declare function readOpenAIVideoFailureMessage(payload: OpenAIVideoResponse): string | undefined;
|
||||||
|
export declare function resolveDurationSeconds(durationSeconds: number | undefined): "4" | "8" | "12" | undefined;
|
||||||
|
export declare function resolveSize(params: {
|
||||||
|
size?: string;
|
||||||
|
aspectRatio?: string;
|
||||||
|
resolution?: string;
|
||||||
|
}): (typeof OPENAI_VIDEO_SIZES)[number] | undefined;
|
||||||
|
export declare function resolveReferenceAsset(req: VideoGenerationRequest): OpenAIReferenceAsset | null;
|
||||||
|
export declare function fetchOpenAIVideoDownload(params: {
|
||||||
|
url: string;
|
||||||
|
init: RequestInit;
|
||||||
|
deadline: ReturnType<typeof createProviderOperationDeadline>;
|
||||||
|
fetchFn: typeof fetch;
|
||||||
|
} & OpenAIVideoRequestPolicy): Promise<any>;
|
||||||
|
export declare function buildOpenAIVideoGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }: {
|
||||||
|
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
|
||||||
|
getPluginConfig?: () => Record<string, unknown> | undefined;
|
||||||
|
}): VideoGenerationProvider;
|
||||||
|
export {};
|
||||||
Vendored
+382
@@ -0,0 +1,382 @@
|
|||||||
|
import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime";
|
||||||
|
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
|
||||||
|
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||||
|
const DEFAULT_OPENAI_VIDEO_BASE_URL = "https://api.openai.com/v1";
|
||||||
|
const DEFAULT_OPENAI_VIDEO_MODEL = "sora-2";
|
||||||
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||||
|
const POLL_INTERVAL_MS = 2_500;
|
||||||
|
const MAX_POLL_ATTEMPTS = 120;
|
||||||
|
const OPENAI_VIDEO_SECONDS = [4, 8, 12];
|
||||||
|
const OPENAI_VIDEO_SIZES = ["720x1280", "1280x720", "1024x1792", "1792x1024"];
|
||||||
|
export function resolvePluginApiKey(rawKey) {
|
||||||
|
if (typeof rawKey === "string") {
|
||||||
|
return normalizeOptionalString(rawKey);
|
||||||
|
}
|
||||||
|
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
|
||||||
|
return normalizeOptionalString(process.env[rawKey.id]);
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
export function resolveVideoConfig(cfg, apiPluginConfig) {
|
||||||
|
const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config);
|
||||||
|
const legacyConfig = cfg?.plugins?.entries?.["openai-video"]?.config;
|
||||||
|
const directVideoConfig = toolsConfig?.video;
|
||||||
|
const flatToolsConfig = toolsConfig?.baseUrl
|
||||||
|
? {
|
||||||
|
baseUrl: toolsConfig.baseUrl,
|
||||||
|
apiKey: toolsConfig.apiKey,
|
||||||
|
allowPrivateNetwork: toolsConfig.allowPrivateNetwork,
|
||||||
|
defaultModel: toolsConfig.defaultModel,
|
||||||
|
models: toolsConfig.models,
|
||||||
|
}
|
||||||
|
: undefined;
|
||||||
|
return {
|
||||||
|
...legacyConfig,
|
||||||
|
...flatToolsConfig,
|
||||||
|
...directVideoConfig,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
export function resolveConfiguredOpenAIBaseUrl(cfg, pluginCfg) {
|
||||||
|
return (normalizeOptionalString(pluginCfg?.baseUrl) ??
|
||||||
|
normalizeOptionalString(process.env.OPENAI_VIDEO_BASE_URL) ??
|
||||||
|
normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ??
|
||||||
|
normalizeOptionalString(process.env.OPENAI_BASE_URL) ??
|
||||||
|
DEFAULT_OPENAI_VIDEO_BASE_URL);
|
||||||
|
}
|
||||||
|
export function readOpenAIVideoFailureMessage(payload) {
|
||||||
|
if (payload.status !== "failed") {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
if (typeof payload.error === "string") {
|
||||||
|
return normalizeOptionalString(payload.error) ?? "OpenAI video generation failed";
|
||||||
|
}
|
||||||
|
return normalizeOptionalString(payload.error?.message) ?? "OpenAI video generation failed";
|
||||||
|
}
|
||||||
|
export function resolveDurationSeconds(durationSeconds) {
|
||||||
|
if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const rounded = Math.max(OPENAI_VIDEO_SECONDS[0], Math.round(durationSeconds));
|
||||||
|
const nearest = OPENAI_VIDEO_SECONDS.reduce((best, current) => Math.abs(current - rounded) < Math.abs(best - rounded) ? current : best);
|
||||||
|
return String(nearest);
|
||||||
|
}
|
||||||
|
export function resolveSize(params) {
|
||||||
|
const explicitSize = normalizeOptionalString(params.size);
|
||||||
|
const supportedSize = OPENAI_VIDEO_SIZES.find((size) => size === explicitSize);
|
||||||
|
if (supportedSize) {
|
||||||
|
return supportedSize;
|
||||||
|
}
|
||||||
|
switch (normalizeOptionalString(params.aspectRatio)) {
|
||||||
|
case "9:16":
|
||||||
|
return "720x1280";
|
||||||
|
case "16:9":
|
||||||
|
return "1280x720";
|
||||||
|
case "4:7":
|
||||||
|
return "1024x1792";
|
||||||
|
case "7:4":
|
||||||
|
return "1792x1024";
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (params.resolution === "1080P") {
|
||||||
|
return "1792x1024";
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
export function resolveReferenceAsset(req) {
|
||||||
|
const allAssets = [...(req.inputImages ?? []), ...(req.inputVideos ?? [])];
|
||||||
|
if (allAssets.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (allAssets.length > 1) {
|
||||||
|
throw new Error("OpenAI video generation supports at most one reference image or video.");
|
||||||
|
}
|
||||||
|
const [asset] = allAssets;
|
||||||
|
if (!asset?.buffer) {
|
||||||
|
throw new Error("OpenAI video generation currently requires local image/video uploads for reference assets.");
|
||||||
|
}
|
||||||
|
const kind = (req.inputVideos?.length ?? 0) > 0 ? "video" : "image";
|
||||||
|
const mimeType = normalizeOptionalString(asset.mimeType) || (kind === "video" ? "video/mp4" : "image/png");
|
||||||
|
const extension = extensionForMime(mimeType)?.slice(1) ?? (mimeType.startsWith("video/") ? "mp4" : "png");
|
||||||
|
const fileName = normalizeOptionalString(asset.fileName) || `reference-${kind}.${extension}`;
|
||||||
|
return {
|
||||||
|
kind,
|
||||||
|
file: new File([bufferToBlobPart(asset.buffer)], fileName, { type: mimeType }),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
export async function fetchOpenAIVideoDownload(params) {
|
||||||
|
const { assertOkOrThrowHttpError, createProviderOperationTimeoutResolver, executeProviderOperationWithRetry, fetchProviderDownloadResponse, fetchWithTimeoutGuarded, } = await import("openclaw/plugin-sdk/provider-http");
|
||||||
|
const timeoutMs = createProviderOperationTimeoutResolver({
|
||||||
|
deadline: params.deadline,
|
||||||
|
defaultTimeoutMs: params.deadline.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||||
|
});
|
||||||
|
if (!params.allowPrivateNetwork && !params.dispatcherPolicy) {
|
||||||
|
const response = await fetchProviderDownloadResponse({
|
||||||
|
url: params.url,
|
||||||
|
init: params.init,
|
||||||
|
deadline: params.deadline,
|
||||||
|
fetchFn: params.fetchFn,
|
||||||
|
provider: "openai",
|
||||||
|
requestFailedMessage: "OpenAI video download failed",
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
response,
|
||||||
|
release: async () => { },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return await executeProviderOperationWithRetry({
|
||||||
|
provider: "openai",
|
||||||
|
stage: "download",
|
||||||
|
operation: async () => {
|
||||||
|
const result = await fetchWithTimeoutGuarded(params.url, params.init, timeoutMs(), params.fetchFn, {
|
||||||
|
...(params.allowPrivateNetwork ? { ssrfPolicy: { allowPrivateNetwork: true } } : {}),
|
||||||
|
...(params.dispatcherPolicy ? { dispatcherPolicy: params.dispatcherPolicy } : {}),
|
||||||
|
auditContext: "openai-video-download",
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await assertOkOrThrowHttpError(result.response, "OpenAI video download failed");
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
await result.release();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
export function buildOpenAIVideoGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }) {
|
||||||
|
return {
|
||||||
|
id: "openai-video",
|
||||||
|
aliases: ["openai"],
|
||||||
|
label: "OpenAI",
|
||||||
|
defaultModel: DEFAULT_OPENAI_VIDEO_MODEL,
|
||||||
|
models: [DEFAULT_OPENAI_VIDEO_MODEL, "sora-2-pro", "gemini-video"],
|
||||||
|
isConfigured: (ctx) => {
|
||||||
|
const pluginCfg = resolveVideoConfig(ctx.cfg, getPluginConfig?.());
|
||||||
|
if (resolvePluginApiKey(pluginCfg?.apiKey) ||
|
||||||
|
Boolean(pluginCfg?.apiKey) ||
|
||||||
|
normalizeOptionalString(process.env.OPENAI_VIDEO_API_KEY) ||
|
||||||
|
normalizeOptionalString(process.env.GEMINI_WEB2API_API_KEY) ||
|
||||||
|
normalizeOptionalString(process.env.CCLOAD_API_KEY)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return isProviderApiKeyConfigured({
|
||||||
|
provider: "openai",
|
||||||
|
...ctx,
|
||||||
|
profileTypes: ["api_key"],
|
||||||
|
});
|
||||||
|
},
|
||||||
|
capabilities: {
|
||||||
|
generate: {
|
||||||
|
maxVideos: 1,
|
||||||
|
maxDurationSeconds: 12,
|
||||||
|
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||||
|
supportsSize: true,
|
||||||
|
sizes: OPENAI_VIDEO_SIZES,
|
||||||
|
},
|
||||||
|
imageToVideo: {
|
||||||
|
enabled: true,
|
||||||
|
maxVideos: 1,
|
||||||
|
maxInputImages: 1,
|
||||||
|
maxDurationSeconds: 12,
|
||||||
|
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||||
|
supportsSize: true,
|
||||||
|
sizes: OPENAI_VIDEO_SIZES,
|
||||||
|
},
|
||||||
|
videoToVideo: {
|
||||||
|
enabled: true,
|
||||||
|
maxVideos: 1,
|
||||||
|
maxInputVideos: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
async generateVideo(req) {
|
||||||
|
const pluginCfg = resolveVideoConfig(req.cfg, getPluginConfig?.());
|
||||||
|
const explicitApiKey = resolvePluginApiKey(pluginCfg?.apiKey) ??
|
||||||
|
normalizeOptionalString(process.env.OPENAI_VIDEO_API_KEY) ??
|
||||||
|
normalizeOptionalString(process.env.GEMINI_WEB2API_API_KEY) ??
|
||||||
|
normalizeOptionalString(process.env.CCLOAD_API_KEY);
|
||||||
|
let apiKey = explicitApiKey;
|
||||||
|
if (!apiKey) {
|
||||||
|
const { resolveApiKeyForProvider } = await import("openclaw/plugin-sdk/provider-auth-runtime");
|
||||||
|
const auth = await resolveApiKeyForProvider({
|
||||||
|
provider: "openai",
|
||||||
|
cfg: req.cfg,
|
||||||
|
agentDir: req.agentDir,
|
||||||
|
store: req.authStore,
|
||||||
|
modelApi: "openai-responses",
|
||||||
|
});
|
||||||
|
if (!auth.apiKey || (auth.mode !== undefined && auth.mode !== "api-key")) {
|
||||||
|
throw new Error("OpenAI API key missing");
|
||||||
|
}
|
||||||
|
apiKey = auth.apiKey;
|
||||||
|
}
|
||||||
|
const [{ assertOkOrThrowHttpError, createProviderOperationDeadline, createProviderOperationTimeoutResolver, pollProviderOperationJson, postJsonRequest, postMultipartRequest, readProviderJsonResponse, resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, sanitizeConfiguredModelProviderRequest, }, { downloadGeneratedVideoAsset, resolveGeneratedMediaMaxBytes },] = await Promise.all([
|
||||||
|
import("openclaw/plugin-sdk/provider-http"),
|
||||||
|
import("openclaw/plugin-sdk/media-generation-runtime"),
|
||||||
|
]);
|
||||||
|
const fetchFn = fetch;
|
||||||
|
const deadline = createProviderOperationDeadline({
|
||||||
|
timeoutMs: req.timeoutMs,
|
||||||
|
label: "OpenAI video generation",
|
||||||
|
});
|
||||||
|
const providerConfig = req.cfg.models?.providers?.openai;
|
||||||
|
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
|
||||||
|
const requestConfig = sanitizeConfiguredModelProviderRequest({
|
||||||
|
...providerConfig?.request,
|
||||||
|
allowPrivateNetwork: allowPrivate,
|
||||||
|
});
|
||||||
|
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({
|
||||||
|
baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg, pluginCfg),
|
||||||
|
defaultBaseUrl: DEFAULT_OPENAI_VIDEO_BASE_URL,
|
||||||
|
allowPrivateNetwork: allowPrivate,
|
||||||
|
request: requestConfig,
|
||||||
|
defaultHeaders: {
|
||||||
|
Authorization: `Bearer ${apiKey}`,
|
||||||
|
},
|
||||||
|
provider: "openai",
|
||||||
|
capability: "video",
|
||||||
|
transport: "http",
|
||||||
|
});
|
||||||
|
const model = normalizeOptionalString(req.model) ??
|
||||||
|
normalizeOptionalString(pluginCfg?.defaultModel) ??
|
||||||
|
DEFAULT_OPENAI_VIDEO_MODEL;
|
||||||
|
const seconds = resolveDurationSeconds(req.durationSeconds);
|
||||||
|
const size = resolveSize(req);
|
||||||
|
const referenceAsset = resolveReferenceAsset(req);
|
||||||
|
const isVideoEdit = referenceAsset?.kind === "video";
|
||||||
|
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
|
||||||
|
deadline,
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
});
|
||||||
|
let response;
|
||||||
|
let release;
|
||||||
|
if (referenceAsset) {
|
||||||
|
const form = new FormData();
|
||||||
|
form.set("prompt", req.prompt);
|
||||||
|
if (isVideoEdit) {
|
||||||
|
form.set("video", referenceAsset.file);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
form.set("model", model);
|
||||||
|
if (seconds) {
|
||||||
|
form.set("seconds", seconds);
|
||||||
|
}
|
||||||
|
if (size) {
|
||||||
|
form.set("size", size);
|
||||||
|
}
|
||||||
|
form.set("input_reference", referenceAsset.file);
|
||||||
|
}
|
||||||
|
const multipartHeaders = new Headers(headers);
|
||||||
|
multipartHeaders.delete("Content-Type");
|
||||||
|
({ response, release } = await postMultipartRequest({
|
||||||
|
url: `${baseUrl}/videos${isVideoEdit ? "/edits" : ""}`,
|
||||||
|
headers: multipartHeaders,
|
||||||
|
body: form,
|
||||||
|
timeoutMs: submitTimeoutMs,
|
||||||
|
fetchFn,
|
||||||
|
allowPrivateNetwork,
|
||||||
|
dispatcherPolicy,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
const jsonHeaders = new Headers(headers);
|
||||||
|
jsonHeaders.set("Content-Type", "application/json");
|
||||||
|
const jsonBody = {
|
||||||
|
model,
|
||||||
|
prompt: req.prompt,
|
||||||
|
...(seconds ? { seconds } : {}),
|
||||||
|
...(size ? { size } : {}),
|
||||||
|
};
|
||||||
|
({ response, release } = await postJsonRequest({
|
||||||
|
url: `${baseUrl}/videos`,
|
||||||
|
headers: jsonHeaders,
|
||||||
|
body: jsonBody,
|
||||||
|
timeoutMs: submitTimeoutMs,
|
||||||
|
fetchFn,
|
||||||
|
allowPrivateNetwork,
|
||||||
|
dispatcherPolicy,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
let submitted;
|
||||||
|
try {
|
||||||
|
await assertOkOrThrowHttpError(response, "OpenAI video generation failed");
|
||||||
|
submitted = await readProviderJsonResponse(response, "OpenAI video generation failed");
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
await release();
|
||||||
|
}
|
||||||
|
const failureMessage = readOpenAIVideoFailureMessage(submitted);
|
||||||
|
if (failureMessage) {
|
||||||
|
throw new Error(failureMessage);
|
||||||
|
}
|
||||||
|
const videoId = normalizeOptionalString(submitted.id);
|
||||||
|
if (!videoId) {
|
||||||
|
throw new Error("OpenAI video generation response missing video id");
|
||||||
|
}
|
||||||
|
const completed = submitted.status === "completed"
|
||||||
|
? submitted
|
||||||
|
: await pollProviderOperationJson({
|
||||||
|
url: `${baseUrl}/videos/${videoId}`,
|
||||||
|
headers,
|
||||||
|
deadline: createProviderOperationDeadline({
|
||||||
|
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||||
|
deadline,
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
}),
|
||||||
|
label: `OpenAI video generation task ${videoId}`,
|
||||||
|
}),
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
fetchFn,
|
||||||
|
maxAttempts: MAX_POLL_ATTEMPTS,
|
||||||
|
pollIntervalMs: POLL_INTERVAL_MS,
|
||||||
|
requestFailedMessage: "OpenAI video status request failed",
|
||||||
|
timeoutMessage: `OpenAI video generation task ${videoId} did not finish in time`,
|
||||||
|
allowPrivateNetwork,
|
||||||
|
dispatcherPolicy,
|
||||||
|
auditContext: "openai-video-status",
|
||||||
|
isComplete: (payload) => payload.status === "completed",
|
||||||
|
getFailureMessage: readOpenAIVideoFailureMessage,
|
||||||
|
});
|
||||||
|
const url = new URL(`${baseUrl}/videos/${videoId}/content`);
|
||||||
|
url.searchParams.set("variant", "video");
|
||||||
|
const video = await downloadGeneratedVideoAsset({
|
||||||
|
url: url.toString(),
|
||||||
|
timeoutMs: createProviderOperationTimeoutResolver({
|
||||||
|
deadline,
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
}),
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
fetchFn,
|
||||||
|
provider: "openai",
|
||||||
|
label: "OpenAI generated video download",
|
||||||
|
requestFailedMessage: "OpenAI video download failed",
|
||||||
|
maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "video"),
|
||||||
|
validateBinaryResponse: true,
|
||||||
|
fetchResponse: async ({ deadline: downloadDeadline }) => await fetchOpenAIVideoDownload({
|
||||||
|
url: url.toString(),
|
||||||
|
init: {
|
||||||
|
method: "GET",
|
||||||
|
headers: new Headers({
|
||||||
|
...Object.fromEntries(headers.entries()),
|
||||||
|
Accept: "application/binary",
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
deadline: downloadDeadline,
|
||||||
|
fetchFn,
|
||||||
|
allowPrivateNetwork,
|
||||||
|
dispatcherPolicy,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
videos: [video],
|
||||||
|
model: completed.model ?? submitted.model ?? model,
|
||||||
|
metadata: {
|
||||||
|
videoId,
|
||||||
|
status: completed.status,
|
||||||
|
seconds: completed.seconds ?? submitted.seconds,
|
||||||
|
size: completed.size ?? submitted.size,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
+132
-7
@@ -1,22 +1,147 @@
|
|||||||
{
|
{
|
||||||
"id": "tools",
|
"id": "tools",
|
||||||
"name": "Tools",
|
"name": "Tools",
|
||||||
"description": "Add Tools tools to OpenClaw.",
|
"description": "Custom Tools and Media Providers for OpenClaw.",
|
||||||
"version": "0.1.0",
|
"version": "0.2.0",
|
||||||
"configSchema": {
|
"categories": [
|
||||||
"type": "object",
|
"media",
|
||||||
"properties": {},
|
"utilities"
|
||||||
"additionalProperties": false
|
],
|
||||||
},
|
|
||||||
"activation": {
|
"activation": {
|
||||||
"onStartup": true
|
"onStartup": true
|
||||||
},
|
},
|
||||||
"skills": [
|
"skills": [
|
||||||
"./skills"
|
"./skills"
|
||||||
],
|
],
|
||||||
|
"setup": {
|
||||||
|
"providers": [
|
||||||
|
{
|
||||||
|
"id": "openai-video",
|
||||||
|
"envVars": [
|
||||||
|
"OPENAI_VIDEO_API_KEY",
|
||||||
|
"CCLOAD_API_KEY",
|
||||||
|
"BIFROST_API_KEY",
|
||||||
|
"OPENAI_API_KEY"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"contracts": {
|
"contracts": {
|
||||||
"tools": [
|
"tools": [
|
||||||
"echo"
|
"echo"
|
||||||
|
],
|
||||||
|
"videoGenerationProviders": [
|
||||||
|
"openai-video"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
"videoGenerationProviderMetadata": {
|
||||||
|
"openai-video": {
|
||||||
|
"authSignals": [
|
||||||
|
{
|
||||||
|
"provider": "openai"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"configContracts": {
|
||||||
|
"secretInputs": {
|
||||||
|
"paths": [
|
||||||
|
{
|
||||||
|
"path": "apiKey",
|
||||||
|
"expected": "string",
|
||||||
|
"ownerKind": "capability"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "video.apiKey",
|
||||||
|
"expected": "string",
|
||||||
|
"ownerKind": "capability"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"uiHints": {
|
||||||
|
"apiKey": {
|
||||||
|
"label": "API Key",
|
||||||
|
"sensitive": true
|
||||||
|
},
|
||||||
|
"video.apiKey": {
|
||||||
|
"label": "Video API Key",
|
||||||
|
"sensitive": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"configSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"baseUrl": {
|
||||||
|
"type": "string",
|
||||||
|
"title": "API Base URL",
|
||||||
|
"description": "Base URL of the OpenAI-compatible video endpoint"
|
||||||
|
},
|
||||||
|
"apiKey": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"object"
|
||||||
|
],
|
||||||
|
"title": "API Key",
|
||||||
|
"description": "API Key or SecretRef for video generation"
|
||||||
|
},
|
||||||
|
"allowPrivateNetwork": {
|
||||||
|
"type": "boolean",
|
||||||
|
"title": "Allow Private Network",
|
||||||
|
"description": "Allow requests to private/internal network endpoints (e.g. host.incus.internal, 127.0.0.1)",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
"defaultModel": {
|
||||||
|
"type": "string",
|
||||||
|
"title": "Default Video Model",
|
||||||
|
"description": "Default model ID to use when no model is explicitly requested",
|
||||||
|
"default": "sora-2"
|
||||||
|
},
|
||||||
|
"models": {
|
||||||
|
"type": "array",
|
||||||
|
"title": "Available Models",
|
||||||
|
"description": "List of available video model IDs",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"default": [
|
||||||
|
"sora-2",
|
||||||
|
"sora-2-pro",
|
||||||
|
"gemini-video"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"video": {
|
||||||
|
"type": "object",
|
||||||
|
"title": "Video Generation Settings",
|
||||||
|
"description": "Nested configuration for video generation",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"baseUrl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apiKey": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"object"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"allowPrivateNetwork": {
|
||||||
|
"type": "boolean",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
"defaultModel": {
|
||||||
|
"type": "string",
|
||||||
|
"default": "sora-2"
|
||||||
|
},
|
||||||
|
"models": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+27
-1
@@ -1,4 +1,4 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import entry from "./index.js";
|
import entry from "./index.js";
|
||||||
import { getToolPluginMetadata } from "openclaw/plugin-sdk/tool-plugin";
|
import { getToolPluginMetadata } from "openclaw/plugin-sdk/tool-plugin";
|
||||||
|
|
||||||
@@ -6,4 +6,30 @@ describe("tools", () => {
|
|||||||
it("declares tool metadata", () => {
|
it("declares tool metadata", () => {
|
||||||
expect(getToolPluginMetadata(entry)?.tools.map((tool) => tool.name)).toEqual(["echo"]);
|
expect(getToolPluginMetadata(entry)?.tools.map((tool) => tool.name)).toEqual(["echo"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("registers both tool and video generation provider", () => {
|
||||||
|
const registeredTools: unknown[] = [];
|
||||||
|
const registeredVideoProviders: unknown[] = [];
|
||||||
|
|
||||||
|
const mockApi = {
|
||||||
|
runtime: {
|
||||||
|
modelAuth: {
|
||||||
|
isProviderApiKeyConfigured: vi.fn().mockReturnValue(false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pluginConfig: {},
|
||||||
|
registerTool: vi.fn((tool) => registeredTools.push(tool)),
|
||||||
|
registerVideoGenerationProvider: vi.fn((provider) =>
|
||||||
|
registeredVideoProviders.push(provider),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
entry.register(mockApi as any);
|
||||||
|
|
||||||
|
expect(mockApi.registerTool).toHaveBeenCalledTimes(1);
|
||||||
|
expect(registeredTools[0]).toMatchObject({ name: "echo" });
|
||||||
|
|
||||||
|
expect(mockApi.registerVideoGenerationProvider).toHaveBeenCalledTimes(1);
|
||||||
|
expect(registeredVideoProviders[0]).toMatchObject({ id: "openai-video" });
|
||||||
|
});
|
||||||
});
|
});
|
||||||
+37
-13
@@ -1,18 +1,42 @@
|
|||||||
import { Type } from "typebox";
|
import { jsonResult } from "openclaw/plugin-sdk/core";
|
||||||
import { defineToolPlugin } from "openclaw/plugin-sdk/tool-plugin";
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||||
|
import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin";
|
||||||
|
import { echoTool } from "./tools/echo.js";
|
||||||
|
import { buildOpenAIVideoGenerationProvider } from "./video/index.js";
|
||||||
|
|
||||||
export default defineToolPlugin({
|
const entry = definePluginEntry({
|
||||||
id: "tools",
|
id: "tools",
|
||||||
name: "Tools",
|
name: "Tools",
|
||||||
description: "Add Tools tools to OpenClaw.",
|
description: "Custom Tools and Media Providers for OpenClaw.",
|
||||||
tools: (tool) => [
|
register(api) {
|
||||||
tool({
|
api.registerTool({
|
||||||
name: "echo",
|
name: echoTool.name,
|
||||||
description: "Echo input text.",
|
label: echoTool.label,
|
||||||
parameters: Type.Object({
|
description: echoTool.description,
|
||||||
input: Type.String({ description: "Text to echo." }),
|
parameters: echoTool.parameters,
|
||||||
|
execute: async (_toolCallId, params) => {
|
||||||
|
const result = await echoTool.execute(params as any);
|
||||||
|
return jsonResult(result);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
api.registerVideoGenerationProvider(
|
||||||
|
buildOpenAIVideoGenerationProvider({
|
||||||
|
isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured,
|
||||||
|
getPluginConfig: () => api.pluginConfig,
|
||||||
}),
|
}),
|
||||||
execute: async ({ input }) => ({ input }),
|
);
|
||||||
}),
|
},
|
||||||
],
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Object.defineProperty(entry, toolPluginMetadataSymbol, {
|
||||||
|
value: {
|
||||||
|
id: "tools",
|
||||||
|
name: "Tools",
|
||||||
|
description: "Custom Tools and Media Providers for OpenClaw.",
|
||||||
|
tools: [echoTool],
|
||||||
|
},
|
||||||
|
enumerable: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
export default entry;
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Type, type Static } from "typebox";
|
||||||
|
|
||||||
|
export const EchoInputSchema = Type.Object({
|
||||||
|
input: Type.String({ description: "Text to echo." }),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type EchoInput = Static<typeof EchoInputSchema>;
|
||||||
|
|
||||||
|
export const echoTool = {
|
||||||
|
name: "echo",
|
||||||
|
label: "Echo",
|
||||||
|
description: "Echo input text.",
|
||||||
|
parameters: EchoInputSchema,
|
||||||
|
execute: async ({ input }: EchoInput) => ({ input }),
|
||||||
|
};
|
||||||
Vendored
+56
@@ -0,0 +1,56 @@
|
|||||||
|
declare module "openclaw/plugin-sdk/blob-runtime" {
|
||||||
|
export function bufferToBlobPart(buffer: Buffer): BlobPart;
|
||||||
|
}
|
||||||
|
|
||||||
|
declare module "openclaw/plugin-sdk/video-generation" {
|
||||||
|
export interface VideoGenerationProvider {
|
||||||
|
id: string;
|
||||||
|
aliases?: string[];
|
||||||
|
label?: string;
|
||||||
|
defaultModel?: string;
|
||||||
|
models?: string[];
|
||||||
|
isConfigured: (ctx: { cfg?: any }) => boolean;
|
||||||
|
capabilities: any;
|
||||||
|
generateVideo: (req: VideoGenerationRequest) => Promise<any>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VideoGenerationRequest {
|
||||||
|
prompt: string;
|
||||||
|
model?: string;
|
||||||
|
durationSeconds?: number;
|
||||||
|
size?: string;
|
||||||
|
aspectRatio?: string;
|
||||||
|
resolution?: string;
|
||||||
|
inputImages?: Array<{ buffer?: Buffer; mimeType?: string; fileName?: string }>;
|
||||||
|
inputVideos?: Array<{ buffer?: Buffer; mimeType?: string; fileName?: string }>;
|
||||||
|
cfg?: any;
|
||||||
|
agentDir?: string;
|
||||||
|
authStore?: any;
|
||||||
|
timeoutMs?: number;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
declare module "openclaw/plugin-sdk/provider-http" {
|
||||||
|
export function createProviderOperationDeadline(params: { timeoutMs?: number; label: string }): any;
|
||||||
|
export function createProviderOperationTimeoutResolver(params: { deadline: any; defaultTimeoutMs: number }): () => number;
|
||||||
|
export function executeProviderOperationWithRetry(params: any): Promise<any>;
|
||||||
|
export function fetchProviderDownloadResponse(params: any): Promise<any>;
|
||||||
|
export function fetchWithTimeoutGuarded(url: string, init: any, timeoutMs: number, fetchFn: any, opts: any): Promise<any>;
|
||||||
|
export function assertOkOrThrowHttpError(response: Response, message: string): Promise<void>;
|
||||||
|
export function postMultipartRequest(params: any): Promise<{ response: Response; release: () => Promise<void> }>;
|
||||||
|
export function postJsonRequest(params: any): Promise<{ response: Response; release: () => Promise<void> }>;
|
||||||
|
export function readProviderJsonResponse<T = any>(response: Response, message: string): Promise<T>;
|
||||||
|
export function pollProviderOperationJson<T = any>(params: any): Promise<T>;
|
||||||
|
export function resolveProviderOperationTimeoutMs(params: any): number;
|
||||||
|
export function resolveProviderHttpRequestConfig(params: any): any;
|
||||||
|
export function sanitizeConfiguredModelProviderRequest(params: any): any;
|
||||||
|
}
|
||||||
|
|
||||||
|
declare module "openclaw/plugin-sdk/provider-auth-runtime" {
|
||||||
|
export function resolveApiKeyForProvider(params: any): Promise<{ apiKey?: string; mode?: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
declare module "openclaw/plugin-sdk/media-generation-runtime" {
|
||||||
|
export function downloadGeneratedVideoAsset(params: any): Promise<any>;
|
||||||
|
export function resolveGeneratedMediaMaxBytes(cfg: any, type: string): number;
|
||||||
|
}
|
||||||
@@ -0,0 +1,519 @@
|
|||||||
|
import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime";
|
||||||
|
import { extensionForMime, type MediaKind } from "openclaw/plugin-sdk/media-mime";
|
||||||
|
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||||
|
import type {
|
||||||
|
createProviderOperationDeadline,
|
||||||
|
postMultipartRequest,
|
||||||
|
} from "openclaw/plugin-sdk/provider-http";
|
||||||
|
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||||
|
import type {
|
||||||
|
VideoGenerationProvider,
|
||||||
|
VideoGenerationRequest,
|
||||||
|
} from "openclaw/plugin-sdk/video-generation";
|
||||||
|
|
||||||
|
const DEFAULT_OPENAI_VIDEO_BASE_URL = "https://api.openai.com/v1";
|
||||||
|
const DEFAULT_OPENAI_VIDEO_MODEL = "sora-2";
|
||||||
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||||
|
const POLL_INTERVAL_MS = 2_500;
|
||||||
|
const MAX_POLL_ATTEMPTS = 120;
|
||||||
|
const OPENAI_VIDEO_SECONDS = [4, 8, 12] as const;
|
||||||
|
const OPENAI_VIDEO_SIZES = ["720x1280", "1280x720", "1024x1792", "1792x1024"] as const;
|
||||||
|
|
||||||
|
export type OpenAIVideoRequestPolicy = {
|
||||||
|
allowPrivateNetwork: boolean;
|
||||||
|
dispatcherPolicy?: Parameters<typeof postMultipartRequest>[0]["dispatcherPolicy"];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type OpenAIVideoStatus = "queued" | "in_progress" | "completed" | "failed";
|
||||||
|
|
||||||
|
export type OpenAIReferenceAsset = {
|
||||||
|
kind: Extract<MediaKind, "image" | "video">;
|
||||||
|
file: File;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type OpenAIVideoResponse = {
|
||||||
|
id?: string;
|
||||||
|
model?: string;
|
||||||
|
status?: OpenAIVideoStatus;
|
||||||
|
prompt?: string | null;
|
||||||
|
seconds?: string;
|
||||||
|
size?: string;
|
||||||
|
error?:
|
||||||
|
| string
|
||||||
|
| {
|
||||||
|
code?: string;
|
||||||
|
message?: string;
|
||||||
|
}
|
||||||
|
| null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface SecretRefObject {
|
||||||
|
source: string;
|
||||||
|
provider: string;
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VideoPluginConfig {
|
||||||
|
baseUrl?: string;
|
||||||
|
apiKey?: string | SecretRefObject;
|
||||||
|
allowPrivateNetwork?: boolean;
|
||||||
|
defaultModel?: string;
|
||||||
|
models?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined {
|
||||||
|
if (typeof rawKey === "string") {
|
||||||
|
return normalizeOptionalString(rawKey);
|
||||||
|
}
|
||||||
|
if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") {
|
||||||
|
return normalizeOptionalString(process.env[rawKey.id]);
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveVideoConfig(
|
||||||
|
cfg: VideoGenerationRequest["cfg"] | undefined,
|
||||||
|
apiPluginConfig?: Record<string, unknown>,
|
||||||
|
): VideoPluginConfig | undefined {
|
||||||
|
const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config) as
|
||||||
|
| { video?: VideoPluginConfig; baseUrl?: string; apiKey?: string | SecretRefObject; defaultModel?: string; models?: string[]; allowPrivateNetwork?: boolean }
|
||||||
|
| undefined;
|
||||||
|
|
||||||
|
const legacyConfig = cfg?.plugins?.entries?.["openai-video"]?.config as VideoPluginConfig | undefined;
|
||||||
|
|
||||||
|
const directVideoConfig = toolsConfig?.video;
|
||||||
|
const flatToolsConfig: VideoPluginConfig | undefined = toolsConfig?.baseUrl
|
||||||
|
? {
|
||||||
|
baseUrl: toolsConfig.baseUrl,
|
||||||
|
apiKey: toolsConfig.apiKey,
|
||||||
|
allowPrivateNetwork: toolsConfig.allowPrivateNetwork,
|
||||||
|
defaultModel: toolsConfig.defaultModel,
|
||||||
|
models: toolsConfig.models,
|
||||||
|
}
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
...legacyConfig,
|
||||||
|
...flatToolsConfig,
|
||||||
|
...directVideoConfig,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveConfiguredOpenAIBaseUrl(
|
||||||
|
cfg: VideoGenerationRequest["cfg"],
|
||||||
|
pluginCfg?: VideoPluginConfig,
|
||||||
|
): string {
|
||||||
|
return (
|
||||||
|
normalizeOptionalString(pluginCfg?.baseUrl) ??
|
||||||
|
normalizeOptionalString(process.env.OPENAI_VIDEO_BASE_URL) ??
|
||||||
|
normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ??
|
||||||
|
normalizeOptionalString(process.env.OPENAI_BASE_URL) ??
|
||||||
|
DEFAULT_OPENAI_VIDEO_BASE_URL
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readOpenAIVideoFailureMessage(payload: OpenAIVideoResponse): string | undefined {
|
||||||
|
if (payload.status !== "failed") {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
if (typeof payload.error === "string") {
|
||||||
|
return normalizeOptionalString(payload.error) ?? "OpenAI video generation failed";
|
||||||
|
}
|
||||||
|
return normalizeOptionalString(payload.error?.message) ?? "OpenAI video generation failed";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveDurationSeconds(durationSeconds: number | undefined): "4" | "8" | "12" | undefined {
|
||||||
|
if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const rounded = Math.max(OPENAI_VIDEO_SECONDS[0], Math.round(durationSeconds));
|
||||||
|
const nearest = OPENAI_VIDEO_SECONDS.reduce((best, current) =>
|
||||||
|
Math.abs(current - rounded) < Math.abs(best - rounded) ? current : best,
|
||||||
|
);
|
||||||
|
return String(nearest) as "4" | "8" | "12";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveSize(params: {
|
||||||
|
size?: string;
|
||||||
|
aspectRatio?: string;
|
||||||
|
resolution?: string;
|
||||||
|
}): (typeof OPENAI_VIDEO_SIZES)[number] | undefined {
|
||||||
|
const explicitSize = normalizeOptionalString(params.size);
|
||||||
|
const supportedSize = OPENAI_VIDEO_SIZES.find((size) => size === explicitSize);
|
||||||
|
if (supportedSize) {
|
||||||
|
return supportedSize;
|
||||||
|
}
|
||||||
|
switch (normalizeOptionalString(params.aspectRatio)) {
|
||||||
|
case "9:16":
|
||||||
|
return "720x1280";
|
||||||
|
case "16:9":
|
||||||
|
return "1280x720";
|
||||||
|
case "4:7":
|
||||||
|
return "1024x1792";
|
||||||
|
case "7:4":
|
||||||
|
return "1792x1024";
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (params.resolution === "1080P") {
|
||||||
|
return "1792x1024";
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveReferenceAsset(req: VideoGenerationRequest): OpenAIReferenceAsset | null {
|
||||||
|
const allAssets = [...(req.inputImages ?? []), ...(req.inputVideos ?? [])];
|
||||||
|
if (allAssets.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (allAssets.length > 1) {
|
||||||
|
throw new Error("OpenAI video generation supports at most one reference image or video.");
|
||||||
|
}
|
||||||
|
const [asset] = allAssets;
|
||||||
|
if (!asset?.buffer) {
|
||||||
|
throw new Error(
|
||||||
|
"OpenAI video generation currently requires local image/video uploads for reference assets.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const kind = (req.inputVideos?.length ?? 0) > 0 ? "video" : "image";
|
||||||
|
const mimeType =
|
||||||
|
normalizeOptionalString(asset.mimeType) || (kind === "video" ? "video/mp4" : "image/png");
|
||||||
|
const extension =
|
||||||
|
extensionForMime(mimeType)?.slice(1) ?? (mimeType.startsWith("video/") ? "mp4" : "png");
|
||||||
|
const fileName = normalizeOptionalString(asset.fileName) || `reference-${kind}.${extension}`;
|
||||||
|
return {
|
||||||
|
kind,
|
||||||
|
file: new File([bufferToBlobPart(asset.buffer)], fileName, { type: mimeType }),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchOpenAIVideoDownload(
|
||||||
|
params: {
|
||||||
|
url: string;
|
||||||
|
init: RequestInit;
|
||||||
|
deadline: ReturnType<typeof createProviderOperationDeadline>;
|
||||||
|
fetchFn: typeof fetch;
|
||||||
|
} & OpenAIVideoRequestPolicy,
|
||||||
|
) {
|
||||||
|
const {
|
||||||
|
assertOkOrThrowHttpError,
|
||||||
|
createProviderOperationTimeoutResolver,
|
||||||
|
executeProviderOperationWithRetry,
|
||||||
|
fetchProviderDownloadResponse,
|
||||||
|
fetchWithTimeoutGuarded,
|
||||||
|
} = await import("openclaw/plugin-sdk/provider-http");
|
||||||
|
const timeoutMs = createProviderOperationTimeoutResolver({
|
||||||
|
deadline: params.deadline,
|
||||||
|
defaultTimeoutMs: params.deadline.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||||
|
});
|
||||||
|
if (!params.allowPrivateNetwork && !params.dispatcherPolicy) {
|
||||||
|
const response = await fetchProviderDownloadResponse({
|
||||||
|
url: params.url,
|
||||||
|
init: params.init,
|
||||||
|
deadline: params.deadline,
|
||||||
|
fetchFn: params.fetchFn,
|
||||||
|
provider: "openai",
|
||||||
|
requestFailedMessage: "OpenAI video download failed",
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
response,
|
||||||
|
release: async () => {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return await executeProviderOperationWithRetry({
|
||||||
|
provider: "openai",
|
||||||
|
stage: "download",
|
||||||
|
operation: async () => {
|
||||||
|
const result = await fetchWithTimeoutGuarded(
|
||||||
|
params.url,
|
||||||
|
params.init,
|
||||||
|
timeoutMs(),
|
||||||
|
params.fetchFn,
|
||||||
|
{
|
||||||
|
...(params.allowPrivateNetwork ? { ssrfPolicy: { allowPrivateNetwork: true } } : {}),
|
||||||
|
...(params.dispatcherPolicy ? { dispatcherPolicy: params.dispatcherPolicy } : {}),
|
||||||
|
auditContext: "openai-video-download",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await assertOkOrThrowHttpError(result.response, "OpenAI video download failed");
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
await result.release();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildOpenAIVideoGenerationProvider({
|
||||||
|
isProviderApiKeyConfigured,
|
||||||
|
getPluginConfig,
|
||||||
|
}: {
|
||||||
|
isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"];
|
||||||
|
getPluginConfig?: () => Record<string, unknown> | undefined;
|
||||||
|
}): VideoGenerationProvider {
|
||||||
|
return {
|
||||||
|
id: "openai-video",
|
||||||
|
aliases: ["openai"],
|
||||||
|
label: "OpenAI",
|
||||||
|
defaultModel: DEFAULT_OPENAI_VIDEO_MODEL,
|
||||||
|
models: [DEFAULT_OPENAI_VIDEO_MODEL, "sora-2-pro", "gemini-video"],
|
||||||
|
isConfigured: (ctx) => {
|
||||||
|
const pluginCfg = resolveVideoConfig(ctx.cfg, getPluginConfig?.());
|
||||||
|
if (
|
||||||
|
resolvePluginApiKey(pluginCfg?.apiKey) ||
|
||||||
|
Boolean(pluginCfg?.apiKey) ||
|
||||||
|
normalizeOptionalString(process.env.OPENAI_VIDEO_API_KEY) ||
|
||||||
|
normalizeOptionalString(process.env.GEMINI_WEB2API_API_KEY) ||
|
||||||
|
normalizeOptionalString(process.env.CCLOAD_API_KEY)
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return isProviderApiKeyConfigured({
|
||||||
|
provider: "openai",
|
||||||
|
...ctx,
|
||||||
|
profileTypes: ["api_key"],
|
||||||
|
});
|
||||||
|
},
|
||||||
|
capabilities: {
|
||||||
|
generate: {
|
||||||
|
maxVideos: 1,
|
||||||
|
maxDurationSeconds: 12,
|
||||||
|
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||||
|
supportsSize: true,
|
||||||
|
sizes: OPENAI_VIDEO_SIZES,
|
||||||
|
},
|
||||||
|
imageToVideo: {
|
||||||
|
enabled: true,
|
||||||
|
maxVideos: 1,
|
||||||
|
maxInputImages: 1,
|
||||||
|
maxDurationSeconds: 12,
|
||||||
|
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||||
|
supportsSize: true,
|
||||||
|
sizes: OPENAI_VIDEO_SIZES,
|
||||||
|
},
|
||||||
|
videoToVideo: {
|
||||||
|
enabled: true,
|
||||||
|
maxVideos: 1,
|
||||||
|
maxInputVideos: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
async generateVideo(req) {
|
||||||
|
const pluginCfg = resolveVideoConfig(req.cfg, getPluginConfig?.());
|
||||||
|
const explicitApiKey =
|
||||||
|
resolvePluginApiKey(pluginCfg?.apiKey) ??
|
||||||
|
normalizeOptionalString(process.env.OPENAI_VIDEO_API_KEY) ??
|
||||||
|
normalizeOptionalString(process.env.GEMINI_WEB2API_API_KEY) ??
|
||||||
|
normalizeOptionalString(process.env.CCLOAD_API_KEY);
|
||||||
|
|
||||||
|
let apiKey = explicitApiKey;
|
||||||
|
if (!apiKey) {
|
||||||
|
const { resolveApiKeyForProvider } =
|
||||||
|
await import("openclaw/plugin-sdk/provider-auth-runtime");
|
||||||
|
const auth = await resolveApiKeyForProvider({
|
||||||
|
provider: "openai",
|
||||||
|
cfg: req.cfg,
|
||||||
|
agentDir: req.agentDir,
|
||||||
|
store: req.authStore,
|
||||||
|
modelApi: "openai-responses",
|
||||||
|
});
|
||||||
|
if (!auth.apiKey || (auth.mode !== undefined && auth.mode !== "api-key")) {
|
||||||
|
throw new Error("OpenAI API key missing");
|
||||||
|
}
|
||||||
|
apiKey = auth.apiKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
const [
|
||||||
|
{
|
||||||
|
assertOkOrThrowHttpError,
|
||||||
|
createProviderOperationDeadline,
|
||||||
|
createProviderOperationTimeoutResolver,
|
||||||
|
pollProviderOperationJson,
|
||||||
|
postJsonRequest,
|
||||||
|
postMultipartRequest,
|
||||||
|
readProviderJsonResponse,
|
||||||
|
resolveProviderOperationTimeoutMs,
|
||||||
|
resolveProviderHttpRequestConfig,
|
||||||
|
sanitizeConfiguredModelProviderRequest,
|
||||||
|
},
|
||||||
|
{ downloadGeneratedVideoAsset, resolveGeneratedMediaMaxBytes },
|
||||||
|
] = await Promise.all([
|
||||||
|
import("openclaw/plugin-sdk/provider-http"),
|
||||||
|
import("openclaw/plugin-sdk/media-generation-runtime"),
|
||||||
|
]);
|
||||||
|
const fetchFn = fetch;
|
||||||
|
const deadline = createProviderOperationDeadline({
|
||||||
|
timeoutMs: req.timeoutMs,
|
||||||
|
label: "OpenAI video generation",
|
||||||
|
});
|
||||||
|
const providerConfig = req.cfg.models?.providers?.openai;
|
||||||
|
const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true;
|
||||||
|
const requestConfig = sanitizeConfiguredModelProviderRequest({
|
||||||
|
...providerConfig?.request,
|
||||||
|
allowPrivateNetwork: allowPrivate,
|
||||||
|
});
|
||||||
|
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||||
|
resolveProviderHttpRequestConfig({
|
||||||
|
baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg, pluginCfg),
|
||||||
|
defaultBaseUrl: DEFAULT_OPENAI_VIDEO_BASE_URL,
|
||||||
|
allowPrivateNetwork: allowPrivate,
|
||||||
|
request: requestConfig,
|
||||||
|
defaultHeaders: {
|
||||||
|
Authorization: `Bearer ${apiKey}`,
|
||||||
|
},
|
||||||
|
provider: "openai",
|
||||||
|
capability: "video",
|
||||||
|
transport: "http",
|
||||||
|
});
|
||||||
|
|
||||||
|
const model =
|
||||||
|
normalizeOptionalString(req.model) ??
|
||||||
|
normalizeOptionalString(pluginCfg?.defaultModel) ??
|
||||||
|
DEFAULT_OPENAI_VIDEO_MODEL;
|
||||||
|
const seconds = resolveDurationSeconds(req.durationSeconds);
|
||||||
|
const size = resolveSize(req);
|
||||||
|
const referenceAsset = resolveReferenceAsset(req);
|
||||||
|
const isVideoEdit = referenceAsset?.kind === "video";
|
||||||
|
const submitTimeoutMs = resolveProviderOperationTimeoutMs({
|
||||||
|
deadline,
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
});
|
||||||
|
|
||||||
|
let response: Response;
|
||||||
|
let release: () => Promise<void>;
|
||||||
|
|
||||||
|
if (referenceAsset) {
|
||||||
|
const form = new FormData();
|
||||||
|
form.set("prompt", req.prompt);
|
||||||
|
if (isVideoEdit) {
|
||||||
|
form.set("video", referenceAsset.file);
|
||||||
|
} else {
|
||||||
|
form.set("model", model);
|
||||||
|
if (seconds) {
|
||||||
|
form.set("seconds", seconds);
|
||||||
|
}
|
||||||
|
if (size) {
|
||||||
|
form.set("size", size);
|
||||||
|
}
|
||||||
|
form.set("input_reference", referenceAsset.file);
|
||||||
|
}
|
||||||
|
const multipartHeaders = new Headers(headers);
|
||||||
|
multipartHeaders.delete("Content-Type");
|
||||||
|
({ response, release } = await postMultipartRequest({
|
||||||
|
url: `${baseUrl}/videos${isVideoEdit ? "/edits" : ""}`,
|
||||||
|
headers: multipartHeaders,
|
||||||
|
body: form,
|
||||||
|
timeoutMs: submitTimeoutMs,
|
||||||
|
fetchFn,
|
||||||
|
allowPrivateNetwork,
|
||||||
|
dispatcherPolicy,
|
||||||
|
}));
|
||||||
|
} else {
|
||||||
|
const jsonHeaders = new Headers(headers);
|
||||||
|
jsonHeaders.set("Content-Type", "application/json");
|
||||||
|
const jsonBody: Record<string, unknown> = {
|
||||||
|
model,
|
||||||
|
prompt: req.prompt,
|
||||||
|
...(seconds ? { seconds } : {}),
|
||||||
|
...(size ? { size } : {}),
|
||||||
|
};
|
||||||
|
({ response, release } = await postJsonRequest({
|
||||||
|
url: `${baseUrl}/videos`,
|
||||||
|
headers: jsonHeaders,
|
||||||
|
body: jsonBody,
|
||||||
|
timeoutMs: submitTimeoutMs,
|
||||||
|
fetchFn,
|
||||||
|
allowPrivateNetwork,
|
||||||
|
dispatcherPolicy,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let submitted: OpenAIVideoResponse;
|
||||||
|
try {
|
||||||
|
await assertOkOrThrowHttpError(response, "OpenAI video generation failed");
|
||||||
|
submitted = await readProviderJsonResponse<OpenAIVideoResponse>(
|
||||||
|
response,
|
||||||
|
"OpenAI video generation failed",
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await release();
|
||||||
|
}
|
||||||
|
const failureMessage = readOpenAIVideoFailureMessage(submitted);
|
||||||
|
if (failureMessage) {
|
||||||
|
throw new Error(failureMessage);
|
||||||
|
}
|
||||||
|
const videoId = normalizeOptionalString(submitted.id);
|
||||||
|
if (!videoId) {
|
||||||
|
throw new Error("OpenAI video generation response missing video id");
|
||||||
|
}
|
||||||
|
const completed =
|
||||||
|
submitted.status === "completed"
|
||||||
|
? submitted
|
||||||
|
: await pollProviderOperationJson<OpenAIVideoResponse>({
|
||||||
|
url: `${baseUrl}/videos/${videoId}`,
|
||||||
|
headers,
|
||||||
|
deadline: createProviderOperationDeadline({
|
||||||
|
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||||
|
deadline,
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
}),
|
||||||
|
label: `OpenAI video generation task ${videoId}`,
|
||||||
|
}),
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
fetchFn,
|
||||||
|
maxAttempts: MAX_POLL_ATTEMPTS,
|
||||||
|
pollIntervalMs: POLL_INTERVAL_MS,
|
||||||
|
requestFailedMessage: "OpenAI video status request failed",
|
||||||
|
timeoutMessage: `OpenAI video generation task ${videoId} did not finish in time`,
|
||||||
|
allowPrivateNetwork,
|
||||||
|
dispatcherPolicy,
|
||||||
|
auditContext: "openai-video-status",
|
||||||
|
isComplete: (payload: OpenAIVideoResponse) => payload.status === "completed",
|
||||||
|
getFailureMessage: readOpenAIVideoFailureMessage,
|
||||||
|
});
|
||||||
|
const url = new URL(`${baseUrl}/videos/${videoId}/content`);
|
||||||
|
url.searchParams.set("variant", "video");
|
||||||
|
const video = await downloadGeneratedVideoAsset({
|
||||||
|
url: url.toString(),
|
||||||
|
timeoutMs: createProviderOperationTimeoutResolver({
|
||||||
|
deadline,
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
}),
|
||||||
|
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||||
|
fetchFn,
|
||||||
|
provider: "openai",
|
||||||
|
label: "OpenAI generated video download",
|
||||||
|
requestFailedMessage: "OpenAI video download failed",
|
||||||
|
maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "video"),
|
||||||
|
validateBinaryResponse: true,
|
||||||
|
fetchResponse: async ({ deadline: downloadDeadline }: { deadline: any }) =>
|
||||||
|
await fetchOpenAIVideoDownload({
|
||||||
|
url: url.toString(),
|
||||||
|
init: {
|
||||||
|
method: "GET",
|
||||||
|
headers: new Headers({
|
||||||
|
...Object.fromEntries(headers.entries()),
|
||||||
|
Accept: "application/binary",
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
deadline: downloadDeadline,
|
||||||
|
fetchFn,
|
||||||
|
allowPrivateNetwork,
|
||||||
|
dispatcherPolicy,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
videos: [video],
|
||||||
|
model: completed.model ?? submitted.model ?? model,
|
||||||
|
metadata: {
|
||||||
|
videoId,
|
||||||
|
status: completed.status,
|
||||||
|
seconds: completed.seconds ?? submitted.seconds,
|
||||||
|
size: completed.size ?? submitted.size,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
+4
-1
@@ -10,6 +10,9 @@
|
|||||||
"skipLibCheck": true
|
"skipLibCheck": true
|
||||||
},
|
},
|
||||||
"include": [
|
"include": [
|
||||||
"src/index.ts"
|
"src/**/*"
|
||||||
|
],
|
||||||
|
"exclude": [
|
||||||
|
"src/**/*.test.ts"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user