diff --git a/dist/index.d.ts b/dist/index.d.ts index aba21d0..81f6087 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,2 +1,14 @@ -declare const _default: import("openclaw/plugin-sdk/tool-plugin").DefinedToolPluginEntry; -export default _default; +declare const entry: Omit<{ + id: string; + name: string; + description: string; + kind?: import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["kind"]; + configSchema?: import("openclaw/plugin-sdk/core").OpenClawPluginConfigSchema | (() => import("openclaw/plugin-sdk/core").OpenClawPluginConfigSchema); + reload?: import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["reload"]; + nodeHostCommands?: import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["nodeHostCommands"]; + securityAuditCollectors?: import("openclaw/plugin-sdk/core").OpenClawPluginDefinition["securityAuditCollectors"]; + register: NonNullable; +}, "configSchema"> & { + configSchema: import("openclaw/plugin-sdk/core").OpenClawPluginConfigSchema; +}; +export default entry; diff --git a/dist/index.js b/dist/index.js index 466b390..6da5f92 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,17 +1,36 @@ -import { Type } from "typebox"; -import { defineToolPlugin } from "openclaw/plugin-sdk/tool-plugin"; -export default defineToolPlugin({ +import { jsonResult } from "openclaw/plugin-sdk/core"; +import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin"; +import { echoTool } from "./tools/echo.js"; +import { buildOpenAIVideoGenerationProvider } from "./video/index.js"; +const entry = definePluginEntry({ id: "tools", name: "Tools", - description: "Add Tools tools to OpenClaw.", - tools: (tool) => [ - tool({ - name: "echo", - description: "Echo input text.", - parameters: Type.Object({ - input: Type.String({ description: "Text to echo." }), - }), - execute: async ({ input }) => ({ input }), - }), - ], + description: "Custom Tools and Media Providers for OpenClaw.", + register(api) { + api.registerTool({ + name: echoTool.name, + label: echoTool.label, + description: echoTool.description, + parameters: echoTool.parameters, + execute: async (_toolCallId, params) => { + const result = await echoTool.execute(params); + return jsonResult(result); + }, + }); + api.registerVideoGenerationProvider(buildOpenAIVideoGenerationProvider({ + isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured, + getPluginConfig: () => api.pluginConfig, + })); + }, }); +Object.defineProperty(entry, toolPluginMetadataSymbol, { + value: { + id: "tools", + name: "Tools", + description: "Custom Tools and Media Providers for OpenClaw.", + tools: [echoTool], + }, + enumerable: false, +}); +export default entry; diff --git a/dist/tools/echo.d.ts b/dist/tools/echo.d.ts new file mode 100644 index 0000000..d31a064 --- /dev/null +++ b/dist/tools/echo.d.ts @@ -0,0 +1,16 @@ +import { Type, type Static } from "typebox"; +export declare const EchoInputSchema: Type.TObject<{ + input: Type.TString; +}>; +export type EchoInput = Static; +export declare const echoTool: { + name: string; + label: string; + description: string; + parameters: Type.TObject<{ + input: Type.TString; + }>; + execute: ({ input }: EchoInput) => Promise<{ + input: string; + }>; +}; diff --git a/dist/tools/echo.js b/dist/tools/echo.js new file mode 100644 index 0000000..0d2357a --- /dev/null +++ b/dist/tools/echo.js @@ -0,0 +1,11 @@ +import { Type } from "typebox"; +export const EchoInputSchema = Type.Object({ + input: Type.String({ description: "Text to echo." }), +}); +export const echoTool = { + name: "echo", + label: "Echo", + description: "Echo input text.", + parameters: EchoInputSchema, + execute: async ({ input }) => ({ input }), +}; diff --git a/dist/video/index.d.ts b/dist/video/index.d.ts new file mode 100644 index 0000000..e3aa016 --- /dev/null +++ b/dist/video/index.d.ts @@ -0,0 +1,60 @@ +import { type MediaKind } from "openclaw/plugin-sdk/media-mime"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import type { createProviderOperationDeadline, postMultipartRequest } from "openclaw/plugin-sdk/provider-http"; +import type { VideoGenerationProvider, VideoGenerationRequest } from "openclaw/plugin-sdk/video-generation"; +declare const OPENAI_VIDEO_SIZES: readonly ["720x1280", "1280x720", "1024x1792", "1792x1024"]; +export type OpenAIVideoRequestPolicy = { + allowPrivateNetwork: boolean; + dispatcherPolicy?: Parameters[0]["dispatcherPolicy"]; +}; +export type OpenAIVideoStatus = "queued" | "in_progress" | "completed" | "failed"; +export type OpenAIReferenceAsset = { + kind: Extract; + file: File; +}; +export type OpenAIVideoResponse = { + id?: string; + model?: string; + status?: OpenAIVideoStatus; + prompt?: string | null; + seconds?: string; + size?: string; + error?: string | { + code?: string; + message?: string; + } | null; +}; +export interface SecretRefObject { + source: string; + provider: string; + id: string; +} +export interface VideoPluginConfig { + baseUrl?: string; + apiKey?: string | SecretRefObject; + allowPrivateNetwork?: boolean; + defaultModel?: string; + models?: string[]; +} +export declare function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined; +export declare function resolveVideoConfig(cfg: VideoGenerationRequest["cfg"] | undefined, apiPluginConfig?: Record): VideoPluginConfig | undefined; +export declare function resolveConfiguredOpenAIBaseUrl(cfg: VideoGenerationRequest["cfg"], pluginCfg?: VideoPluginConfig): string; +export declare function readOpenAIVideoFailureMessage(payload: OpenAIVideoResponse): string | undefined; +export declare function resolveDurationSeconds(durationSeconds: number | undefined): "4" | "8" | "12" | undefined; +export declare function resolveSize(params: { + size?: string; + aspectRatio?: string; + resolution?: string; +}): (typeof OPENAI_VIDEO_SIZES)[number] | undefined; +export declare function resolveReferenceAsset(req: VideoGenerationRequest): OpenAIReferenceAsset | null; +export declare function fetchOpenAIVideoDownload(params: { + url: string; + init: RequestInit; + deadline: ReturnType; + fetchFn: typeof fetch; +} & OpenAIVideoRequestPolicy): Promise; +export declare function buildOpenAIVideoGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }: { + isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"]; + getPluginConfig?: () => Record | undefined; +}): VideoGenerationProvider; +export {}; diff --git a/dist/video/index.js b/dist/video/index.js new file mode 100644 index 0000000..a219005 --- /dev/null +++ b/dist/video/index.js @@ -0,0 +1,382 @@ +import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime"; +import { extensionForMime } from "openclaw/plugin-sdk/media-mime"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; +const DEFAULT_OPENAI_VIDEO_BASE_URL = "https://api.openai.com/v1"; +const DEFAULT_OPENAI_VIDEO_MODEL = "sora-2"; +const DEFAULT_TIMEOUT_MS = 120_000; +const POLL_INTERVAL_MS = 2_500; +const MAX_POLL_ATTEMPTS = 120; +const OPENAI_VIDEO_SECONDS = [4, 8, 12]; +const OPENAI_VIDEO_SIZES = ["720x1280", "1280x720", "1024x1792", "1792x1024"]; +export function resolvePluginApiKey(rawKey) { + if (typeof rawKey === "string") { + return normalizeOptionalString(rawKey); + } + if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") { + return normalizeOptionalString(process.env[rawKey.id]); + } + return undefined; +} +export function resolveVideoConfig(cfg, apiPluginConfig) { + const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config); + const legacyConfig = cfg?.plugins?.entries?.["openai-video"]?.config; + const directVideoConfig = toolsConfig?.video; + const flatToolsConfig = toolsConfig?.baseUrl + ? { + baseUrl: toolsConfig.baseUrl, + apiKey: toolsConfig.apiKey, + allowPrivateNetwork: toolsConfig.allowPrivateNetwork, + defaultModel: toolsConfig.defaultModel, + models: toolsConfig.models, + } + : undefined; + return { + ...legacyConfig, + ...flatToolsConfig, + ...directVideoConfig, + }; +} +export function resolveConfiguredOpenAIBaseUrl(cfg, pluginCfg) { + return (normalizeOptionalString(pluginCfg?.baseUrl) ?? + normalizeOptionalString(process.env.OPENAI_VIDEO_BASE_URL) ?? + normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ?? + normalizeOptionalString(process.env.OPENAI_BASE_URL) ?? + DEFAULT_OPENAI_VIDEO_BASE_URL); +} +export function readOpenAIVideoFailureMessage(payload) { + if (payload.status !== "failed") { + return undefined; + } + if (typeof payload.error === "string") { + return normalizeOptionalString(payload.error) ?? "OpenAI video generation failed"; + } + return normalizeOptionalString(payload.error?.message) ?? "OpenAI video generation failed"; +} +export function resolveDurationSeconds(durationSeconds) { + if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) { + return undefined; + } + const rounded = Math.max(OPENAI_VIDEO_SECONDS[0], Math.round(durationSeconds)); + const nearest = OPENAI_VIDEO_SECONDS.reduce((best, current) => Math.abs(current - rounded) < Math.abs(best - rounded) ? current : best); + return String(nearest); +} +export function resolveSize(params) { + const explicitSize = normalizeOptionalString(params.size); + const supportedSize = OPENAI_VIDEO_SIZES.find((size) => size === explicitSize); + if (supportedSize) { + return supportedSize; + } + switch (normalizeOptionalString(params.aspectRatio)) { + case "9:16": + return "720x1280"; + case "16:9": + return "1280x720"; + case "4:7": + return "1024x1792"; + case "7:4": + return "1792x1024"; + default: + break; + } + if (params.resolution === "1080P") { + return "1792x1024"; + } + return undefined; +} +export function resolveReferenceAsset(req) { + const allAssets = [...(req.inputImages ?? []), ...(req.inputVideos ?? [])]; + if (allAssets.length === 0) { + return null; + } + if (allAssets.length > 1) { + throw new Error("OpenAI video generation supports at most one reference image or video."); + } + const [asset] = allAssets; + if (!asset?.buffer) { + throw new Error("OpenAI video generation currently requires local image/video uploads for reference assets."); + } + const kind = (req.inputVideos?.length ?? 0) > 0 ? "video" : "image"; + const mimeType = normalizeOptionalString(asset.mimeType) || (kind === "video" ? "video/mp4" : "image/png"); + const extension = extensionForMime(mimeType)?.slice(1) ?? (mimeType.startsWith("video/") ? "mp4" : "png"); + const fileName = normalizeOptionalString(asset.fileName) || `reference-${kind}.${extension}`; + return { + kind, + file: new File([bufferToBlobPart(asset.buffer)], fileName, { type: mimeType }), + }; +} +export async function fetchOpenAIVideoDownload(params) { + const { assertOkOrThrowHttpError, createProviderOperationTimeoutResolver, executeProviderOperationWithRetry, fetchProviderDownloadResponse, fetchWithTimeoutGuarded, } = await import("openclaw/plugin-sdk/provider-http"); + const timeoutMs = createProviderOperationTimeoutResolver({ + deadline: params.deadline, + defaultTimeoutMs: params.deadline.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + if (!params.allowPrivateNetwork && !params.dispatcherPolicy) { + const response = await fetchProviderDownloadResponse({ + url: params.url, + init: params.init, + deadline: params.deadline, + fetchFn: params.fetchFn, + provider: "openai", + requestFailedMessage: "OpenAI video download failed", + }); + return { + response, + release: async () => { }, + }; + } + return await executeProviderOperationWithRetry({ + provider: "openai", + stage: "download", + operation: async () => { + const result = await fetchWithTimeoutGuarded(params.url, params.init, timeoutMs(), params.fetchFn, { + ...(params.allowPrivateNetwork ? { ssrfPolicy: { allowPrivateNetwork: true } } : {}), + ...(params.dispatcherPolicy ? { dispatcherPolicy: params.dispatcherPolicy } : {}), + auditContext: "openai-video-download", + }); + try { + await assertOkOrThrowHttpError(result.response, "OpenAI video download failed"); + return result; + } + catch (error) { + await result.release(); + throw error; + } + }, + }); +} +export function buildOpenAIVideoGenerationProvider({ isProviderApiKeyConfigured, getPluginConfig, }) { + return { + id: "openai-video", + aliases: ["openai"], + label: "OpenAI", + defaultModel: DEFAULT_OPENAI_VIDEO_MODEL, + models: [DEFAULT_OPENAI_VIDEO_MODEL, "sora-2-pro", "gemini-video"], + isConfigured: (ctx) => { + const pluginCfg = resolveVideoConfig(ctx.cfg, getPluginConfig?.()); + if (resolvePluginApiKey(pluginCfg?.apiKey) || + Boolean(pluginCfg?.apiKey) || + normalizeOptionalString(process.env.OPENAI_VIDEO_API_KEY) || + normalizeOptionalString(process.env.GEMINI_WEB2API_API_KEY) || + normalizeOptionalString(process.env.CCLOAD_API_KEY)) { + return true; + } + return isProviderApiKeyConfigured({ + provider: "openai", + ...ctx, + profileTypes: ["api_key"], + }); + }, + capabilities: { + generate: { + maxVideos: 1, + maxDurationSeconds: 12, + supportedDurationSeconds: OPENAI_VIDEO_SECONDS, + supportsSize: true, + sizes: OPENAI_VIDEO_SIZES, + }, + imageToVideo: { + enabled: true, + maxVideos: 1, + maxInputImages: 1, + maxDurationSeconds: 12, + supportedDurationSeconds: OPENAI_VIDEO_SECONDS, + supportsSize: true, + sizes: OPENAI_VIDEO_SIZES, + }, + videoToVideo: { + enabled: true, + maxVideos: 1, + maxInputVideos: 1, + }, + }, + async generateVideo(req) { + const pluginCfg = resolveVideoConfig(req.cfg, getPluginConfig?.()); + const explicitApiKey = resolvePluginApiKey(pluginCfg?.apiKey) ?? + normalizeOptionalString(process.env.OPENAI_VIDEO_API_KEY) ?? + normalizeOptionalString(process.env.GEMINI_WEB2API_API_KEY) ?? + normalizeOptionalString(process.env.CCLOAD_API_KEY); + let apiKey = explicitApiKey; + if (!apiKey) { + const { resolveApiKeyForProvider } = await import("openclaw/plugin-sdk/provider-auth-runtime"); + const auth = await resolveApiKeyForProvider({ + provider: "openai", + cfg: req.cfg, + agentDir: req.agentDir, + store: req.authStore, + modelApi: "openai-responses", + }); + if (!auth.apiKey || (auth.mode !== undefined && auth.mode !== "api-key")) { + throw new Error("OpenAI API key missing"); + } + apiKey = auth.apiKey; + } + const [{ assertOkOrThrowHttpError, createProviderOperationDeadline, createProviderOperationTimeoutResolver, pollProviderOperationJson, postJsonRequest, postMultipartRequest, readProviderJsonResponse, resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, sanitizeConfiguredModelProviderRequest, }, { downloadGeneratedVideoAsset, resolveGeneratedMediaMaxBytes },] = await Promise.all([ + import("openclaw/plugin-sdk/provider-http"), + import("openclaw/plugin-sdk/media-generation-runtime"), + ]); + const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "OpenAI video generation", + }); + const providerConfig = req.cfg.models?.providers?.openai; + const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true; + const requestConfig = sanitizeConfiguredModelProviderRequest({ + ...providerConfig?.request, + allowPrivateNetwork: allowPrivate, + }); + const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ + baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg, pluginCfg), + defaultBaseUrl: DEFAULT_OPENAI_VIDEO_BASE_URL, + allowPrivateNetwork: allowPrivate, + request: requestConfig, + defaultHeaders: { + Authorization: `Bearer ${apiKey}`, + }, + provider: "openai", + capability: "video", + transport: "http", + }); + const model = normalizeOptionalString(req.model) ?? + normalizeOptionalString(pluginCfg?.defaultModel) ?? + DEFAULT_OPENAI_VIDEO_MODEL; + const seconds = resolveDurationSeconds(req.durationSeconds); + const size = resolveSize(req); + const referenceAsset = resolveReferenceAsset(req); + const isVideoEdit = referenceAsset?.kind === "video"; + const submitTimeoutMs = resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }); + let response; + let release; + if (referenceAsset) { + const form = new FormData(); + form.set("prompt", req.prompt); + if (isVideoEdit) { + form.set("video", referenceAsset.file); + } + else { + form.set("model", model); + if (seconds) { + form.set("seconds", seconds); + } + if (size) { + form.set("size", size); + } + form.set("input_reference", referenceAsset.file); + } + const multipartHeaders = new Headers(headers); + multipartHeaders.delete("Content-Type"); + ({ response, release } = await postMultipartRequest({ + url: `${baseUrl}/videos${isVideoEdit ? "/edits" : ""}`, + headers: multipartHeaders, + body: form, + timeoutMs: submitTimeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + })); + } + else { + const jsonHeaders = new Headers(headers); + jsonHeaders.set("Content-Type", "application/json"); + const jsonBody = { + model, + prompt: req.prompt, + ...(seconds ? { seconds } : {}), + ...(size ? { size } : {}), + }; + ({ response, release } = await postJsonRequest({ + url: `${baseUrl}/videos`, + headers: jsonHeaders, + body: jsonBody, + timeoutMs: submitTimeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + })); + } + let submitted; + try { + await assertOkOrThrowHttpError(response, "OpenAI video generation failed"); + submitted = await readProviderJsonResponse(response, "OpenAI video generation failed"); + } + finally { + await release(); + } + const failureMessage = readOpenAIVideoFailureMessage(submitted); + if (failureMessage) { + throw new Error(failureMessage); + } + const videoId = normalizeOptionalString(submitted.id); + if (!videoId) { + throw new Error("OpenAI video generation response missing video id"); + } + const completed = submitted.status === "completed" + ? submitted + : await pollProviderOperationJson({ + url: `${baseUrl}/videos/${videoId}`, + headers, + deadline: createProviderOperationDeadline({ + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), + label: `OpenAI video generation task ${videoId}`, + }), + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + fetchFn, + maxAttempts: MAX_POLL_ATTEMPTS, + pollIntervalMs: POLL_INTERVAL_MS, + requestFailedMessage: "OpenAI video status request failed", + timeoutMessage: `OpenAI video generation task ${videoId} did not finish in time`, + allowPrivateNetwork, + dispatcherPolicy, + auditContext: "openai-video-status", + isComplete: (payload) => payload.status === "completed", + getFailureMessage: readOpenAIVideoFailureMessage, + }); + const url = new URL(`${baseUrl}/videos/${videoId}/content`); + url.searchParams.set("variant", "video"); + const video = await downloadGeneratedVideoAsset({ + url: url.toString(), + timeoutMs: createProviderOperationTimeoutResolver({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + fetchFn, + provider: "openai", + label: "OpenAI generated video download", + requestFailedMessage: "OpenAI video download failed", + maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "video"), + validateBinaryResponse: true, + fetchResponse: async ({ deadline: downloadDeadline }) => await fetchOpenAIVideoDownload({ + url: url.toString(), + init: { + method: "GET", + headers: new Headers({ + ...Object.fromEntries(headers.entries()), + Accept: "application/binary", + }), + }, + deadline: downloadDeadline, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + }), + }); + return { + videos: [video], + model: completed.model ?? submitted.model ?? model, + metadata: { + videoId, + status: completed.status, + seconds: completed.seconds ?? submitted.seconds, + size: completed.size ?? submitted.size, + }, + }; + }, + }; +} diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 82201d2..a2b7b7f 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -1,22 +1,147 @@ { "id": "tools", "name": "Tools", - "description": "Add Tools tools to OpenClaw.", - "version": "0.1.0", - "configSchema": { - "type": "object", - "properties": {}, - "additionalProperties": false - }, + "description": "Custom Tools and Media Providers for OpenClaw.", + "version": "0.2.0", + "categories": [ + "media", + "utilities" + ], "activation": { "onStartup": true }, "skills": [ "./skills" ], + "setup": { + "providers": [ + { + "id": "openai-video", + "envVars": [ + "OPENAI_VIDEO_API_KEY", + "CCLOAD_API_KEY", + "BIFROST_API_KEY", + "OPENAI_API_KEY" + ] + } + ] + }, "contracts": { "tools": [ "echo" + ], + "videoGenerationProviders": [ + "openai-video" ] + }, + "videoGenerationProviderMetadata": { + "openai-video": { + "authSignals": [ + { + "provider": "openai" + } + ] + } + }, + "configContracts": { + "secretInputs": { + "paths": [ + { + "path": "apiKey", + "expected": "string", + "ownerKind": "capability" + }, + { + "path": "video.apiKey", + "expected": "string", + "ownerKind": "capability" + } + ] + } + }, + "uiHints": { + "apiKey": { + "label": "API Key", + "sensitive": true + }, + "video.apiKey": { + "label": "Video API Key", + "sensitive": true + } + }, + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": { + "baseUrl": { + "type": "string", + "title": "API Base URL", + "description": "Base URL of the OpenAI-compatible video endpoint" + }, + "apiKey": { + "type": [ + "string", + "object" + ], + "title": "API Key", + "description": "API Key or SecretRef for video generation" + }, + "allowPrivateNetwork": { + "type": "boolean", + "title": "Allow Private Network", + "description": "Allow requests to private/internal network endpoints (e.g. host.incus.internal, 127.0.0.1)", + "default": true + }, + "defaultModel": { + "type": "string", + "title": "Default Video Model", + "description": "Default model ID to use when no model is explicitly requested", + "default": "sora-2" + }, + "models": { + "type": "array", + "title": "Available Models", + "description": "List of available video model IDs", + "items": { + "type": "string" + }, + "default": [ + "sora-2", + "sora-2-pro", + "gemini-video" + ] + }, + "video": { + "type": "object", + "title": "Video Generation Settings", + "description": "Nested configuration for video generation", + "additionalProperties": false, + "properties": { + "baseUrl": { + "type": "string" + }, + "apiKey": { + "type": [ + "string", + "object" + ] + }, + "allowPrivateNetwork": { + "type": "boolean", + "default": true + }, + "defaultModel": { + "type": "string", + "default": "sora-2" + }, + "models": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } } } diff --git a/src/index.test.ts b/src/index.test.ts index 1c66811..9ed153d 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import entry from "./index.js"; import { getToolPluginMetadata } from "openclaw/plugin-sdk/tool-plugin"; @@ -6,4 +6,30 @@ describe("tools", () => { it("declares tool metadata", () => { expect(getToolPluginMetadata(entry)?.tools.map((tool) => tool.name)).toEqual(["echo"]); }); + + it("registers both tool and video generation provider", () => { + const registeredTools: unknown[] = []; + const registeredVideoProviders: unknown[] = []; + + const mockApi = { + runtime: { + modelAuth: { + isProviderApiKeyConfigured: vi.fn().mockReturnValue(false), + }, + }, + pluginConfig: {}, + registerTool: vi.fn((tool) => registeredTools.push(tool)), + registerVideoGenerationProvider: vi.fn((provider) => + registeredVideoProviders.push(provider), + ), + }; + + entry.register(mockApi as any); + + expect(mockApi.registerTool).toHaveBeenCalledTimes(1); + expect(registeredTools[0]).toMatchObject({ name: "echo" }); + + expect(mockApi.registerVideoGenerationProvider).toHaveBeenCalledTimes(1); + expect(registeredVideoProviders[0]).toMatchObject({ id: "openai-video" }); + }); }); diff --git a/src/index.ts b/src/index.ts index d33abea..62089d6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,18 +1,42 @@ -import { Type } from "typebox"; -import { defineToolPlugin } from "openclaw/plugin-sdk/tool-plugin"; +import { jsonResult } from "openclaw/plugin-sdk/core"; +import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { toolPluginMetadataSymbol } from "openclaw/plugin-sdk/tool-plugin"; +import { echoTool } from "./tools/echo.js"; +import { buildOpenAIVideoGenerationProvider } from "./video/index.js"; -export default defineToolPlugin({ +const entry = definePluginEntry({ id: "tools", name: "Tools", - description: "Add Tools tools to OpenClaw.", - tools: (tool) => [ - tool({ - name: "echo", - description: "Echo input text.", - parameters: Type.Object({ - input: Type.String({ description: "Text to echo." }), + description: "Custom Tools and Media Providers for OpenClaw.", + register(api) { + api.registerTool({ + name: echoTool.name, + label: echoTool.label, + description: echoTool.description, + parameters: echoTool.parameters, + execute: async (_toolCallId, params) => { + const result = await echoTool.execute(params as any); + return jsonResult(result); + }, + }); + + api.registerVideoGenerationProvider( + buildOpenAIVideoGenerationProvider({ + isProviderApiKeyConfigured: api.runtime.modelAuth.isProviderApiKeyConfigured, + getPluginConfig: () => api.pluginConfig, }), - execute: async ({ input }) => ({ input }), - }), - ], + ); + }, }); + +Object.defineProperty(entry, toolPluginMetadataSymbol, { + value: { + id: "tools", + name: "Tools", + description: "Custom Tools and Media Providers for OpenClaw.", + tools: [echoTool], + }, + enumerable: false, +}); + +export default entry; diff --git a/src/tools/echo.ts b/src/tools/echo.ts new file mode 100644 index 0000000..1069802 --- /dev/null +++ b/src/tools/echo.ts @@ -0,0 +1,15 @@ +import { Type, type Static } from "typebox"; + +export const EchoInputSchema = Type.Object({ + input: Type.String({ description: "Text to echo." }), +}); + +export type EchoInput = Static; + +export const echoTool = { + name: "echo", + label: "Echo", + description: "Echo input text.", + parameters: EchoInputSchema, + execute: async ({ input }: EchoInput) => ({ input }), +}; diff --git a/src/types.d.ts b/src/types.d.ts new file mode 100644 index 0000000..33f8a63 --- /dev/null +++ b/src/types.d.ts @@ -0,0 +1,56 @@ +declare module "openclaw/plugin-sdk/blob-runtime" { + export function bufferToBlobPart(buffer: Buffer): BlobPart; +} + +declare module "openclaw/plugin-sdk/video-generation" { + export interface VideoGenerationProvider { + id: string; + aliases?: string[]; + label?: string; + defaultModel?: string; + models?: string[]; + isConfigured: (ctx: { cfg?: any }) => boolean; + capabilities: any; + generateVideo: (req: VideoGenerationRequest) => Promise; + } + + export interface VideoGenerationRequest { + prompt: string; + model?: string; + durationSeconds?: number; + size?: string; + aspectRatio?: string; + resolution?: string; + inputImages?: Array<{ buffer?: Buffer; mimeType?: string; fileName?: string }>; + inputVideos?: Array<{ buffer?: Buffer; mimeType?: string; fileName?: string }>; + cfg?: any; + agentDir?: string; + authStore?: any; + timeoutMs?: number; + } +} + +declare module "openclaw/plugin-sdk/provider-http" { + export function createProviderOperationDeadline(params: { timeoutMs?: number; label: string }): any; + export function createProviderOperationTimeoutResolver(params: { deadline: any; defaultTimeoutMs: number }): () => number; + export function executeProviderOperationWithRetry(params: any): Promise; + export function fetchProviderDownloadResponse(params: any): Promise; + export function fetchWithTimeoutGuarded(url: string, init: any, timeoutMs: number, fetchFn: any, opts: any): Promise; + export function assertOkOrThrowHttpError(response: Response, message: string): Promise; + export function postMultipartRequest(params: any): Promise<{ response: Response; release: () => Promise }>; + export function postJsonRequest(params: any): Promise<{ response: Response; release: () => Promise }>; + export function readProviderJsonResponse(response: Response, message: string): Promise; + export function pollProviderOperationJson(params: any): Promise; + export function resolveProviderOperationTimeoutMs(params: any): number; + export function resolveProviderHttpRequestConfig(params: any): any; + export function sanitizeConfiguredModelProviderRequest(params: any): any; +} + +declare module "openclaw/plugin-sdk/provider-auth-runtime" { + export function resolveApiKeyForProvider(params: any): Promise<{ apiKey?: string; mode?: string }>; +} + +declare module "openclaw/plugin-sdk/media-generation-runtime" { + export function downloadGeneratedVideoAsset(params: any): Promise; + export function resolveGeneratedMediaMaxBytes(cfg: any, type: string): number; +} diff --git a/src/video/index.ts b/src/video/index.ts new file mode 100644 index 0000000..b3ea840 --- /dev/null +++ b/src/video/index.ts @@ -0,0 +1,519 @@ +import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime"; +import { extensionForMime, type MediaKind } from "openclaw/plugin-sdk/media-mime"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import type { + createProviderOperationDeadline, + postMultipartRequest, +} from "openclaw/plugin-sdk/provider-http"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; +import type { + VideoGenerationProvider, + VideoGenerationRequest, +} from "openclaw/plugin-sdk/video-generation"; + +const DEFAULT_OPENAI_VIDEO_BASE_URL = "https://api.openai.com/v1"; +const DEFAULT_OPENAI_VIDEO_MODEL = "sora-2"; +const DEFAULT_TIMEOUT_MS = 120_000; +const POLL_INTERVAL_MS = 2_500; +const MAX_POLL_ATTEMPTS = 120; +const OPENAI_VIDEO_SECONDS = [4, 8, 12] as const; +const OPENAI_VIDEO_SIZES = ["720x1280", "1280x720", "1024x1792", "1792x1024"] as const; + +export type OpenAIVideoRequestPolicy = { + allowPrivateNetwork: boolean; + dispatcherPolicy?: Parameters[0]["dispatcherPolicy"]; +}; + +export type OpenAIVideoStatus = "queued" | "in_progress" | "completed" | "failed"; + +export type OpenAIReferenceAsset = { + kind: Extract; + file: File; +}; + +export type OpenAIVideoResponse = { + id?: string; + model?: string; + status?: OpenAIVideoStatus; + prompt?: string | null; + seconds?: string; + size?: string; + error?: + | string + | { + code?: string; + message?: string; + } + | null; +}; + +export interface SecretRefObject { + source: string; + provider: string; + id: string; +} + +export interface VideoPluginConfig { + baseUrl?: string; + apiKey?: string | SecretRefObject; + allowPrivateNetwork?: boolean; + defaultModel?: string; + models?: string[]; +} + +export function resolvePluginApiKey(rawKey: string | SecretRefObject | undefined): string | undefined { + if (typeof rawKey === "string") { + return normalizeOptionalString(rawKey); + } + if (rawKey && typeof rawKey === "object" && typeof rawKey.id === "string") { + return normalizeOptionalString(process.env[rawKey.id]); + } + return undefined; +} + +export function resolveVideoConfig( + cfg: VideoGenerationRequest["cfg"] | undefined, + apiPluginConfig?: Record, +): VideoPluginConfig | undefined { + const toolsConfig = (apiPluginConfig ?? cfg?.plugins?.entries?.["tools"]?.config) as + | { video?: VideoPluginConfig; baseUrl?: string; apiKey?: string | SecretRefObject; defaultModel?: string; models?: string[]; allowPrivateNetwork?: boolean } + | undefined; + + const legacyConfig = cfg?.plugins?.entries?.["openai-video"]?.config as VideoPluginConfig | undefined; + + const directVideoConfig = toolsConfig?.video; + const flatToolsConfig: VideoPluginConfig | undefined = toolsConfig?.baseUrl + ? { + baseUrl: toolsConfig.baseUrl, + apiKey: toolsConfig.apiKey, + allowPrivateNetwork: toolsConfig.allowPrivateNetwork, + defaultModel: toolsConfig.defaultModel, + models: toolsConfig.models, + } + : undefined; + + return { + ...legacyConfig, + ...flatToolsConfig, + ...directVideoConfig, + }; +} + +export function resolveConfiguredOpenAIBaseUrl( + cfg: VideoGenerationRequest["cfg"], + pluginCfg?: VideoPluginConfig, +): string { + return ( + normalizeOptionalString(pluginCfg?.baseUrl) ?? + normalizeOptionalString(process.env.OPENAI_VIDEO_BASE_URL) ?? + normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ?? + normalizeOptionalString(process.env.OPENAI_BASE_URL) ?? + DEFAULT_OPENAI_VIDEO_BASE_URL + ); +} + +export function readOpenAIVideoFailureMessage(payload: OpenAIVideoResponse): string | undefined { + if (payload.status !== "failed") { + return undefined; + } + if (typeof payload.error === "string") { + return normalizeOptionalString(payload.error) ?? "OpenAI video generation failed"; + } + return normalizeOptionalString(payload.error?.message) ?? "OpenAI video generation failed"; +} + +export function resolveDurationSeconds(durationSeconds: number | undefined): "4" | "8" | "12" | undefined { + if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) { + return undefined; + } + const rounded = Math.max(OPENAI_VIDEO_SECONDS[0], Math.round(durationSeconds)); + const nearest = OPENAI_VIDEO_SECONDS.reduce((best, current) => + Math.abs(current - rounded) < Math.abs(best - rounded) ? current : best, + ); + return String(nearest) as "4" | "8" | "12"; +} + +export function resolveSize(params: { + size?: string; + aspectRatio?: string; + resolution?: string; +}): (typeof OPENAI_VIDEO_SIZES)[number] | undefined { + const explicitSize = normalizeOptionalString(params.size); + const supportedSize = OPENAI_VIDEO_SIZES.find((size) => size === explicitSize); + if (supportedSize) { + return supportedSize; + } + switch (normalizeOptionalString(params.aspectRatio)) { + case "9:16": + return "720x1280"; + case "16:9": + return "1280x720"; + case "4:7": + return "1024x1792"; + case "7:4": + return "1792x1024"; + default: + break; + } + if (params.resolution === "1080P") { + return "1792x1024"; + } + return undefined; +} + +export function resolveReferenceAsset(req: VideoGenerationRequest): OpenAIReferenceAsset | null { + const allAssets = [...(req.inputImages ?? []), ...(req.inputVideos ?? [])]; + if (allAssets.length === 0) { + return null; + } + if (allAssets.length > 1) { + throw new Error("OpenAI video generation supports at most one reference image or video."); + } + const [asset] = allAssets; + if (!asset?.buffer) { + throw new Error( + "OpenAI video generation currently requires local image/video uploads for reference assets.", + ); + } + const kind = (req.inputVideos?.length ?? 0) > 0 ? "video" : "image"; + const mimeType = + normalizeOptionalString(asset.mimeType) || (kind === "video" ? "video/mp4" : "image/png"); + const extension = + extensionForMime(mimeType)?.slice(1) ?? (mimeType.startsWith("video/") ? "mp4" : "png"); + const fileName = normalizeOptionalString(asset.fileName) || `reference-${kind}.${extension}`; + return { + kind, + file: new File([bufferToBlobPart(asset.buffer)], fileName, { type: mimeType }), + }; +} + +export async function fetchOpenAIVideoDownload( + params: { + url: string; + init: RequestInit; + deadline: ReturnType; + fetchFn: typeof fetch; + } & OpenAIVideoRequestPolicy, +) { + const { + assertOkOrThrowHttpError, + createProviderOperationTimeoutResolver, + executeProviderOperationWithRetry, + fetchProviderDownloadResponse, + fetchWithTimeoutGuarded, + } = await import("openclaw/plugin-sdk/provider-http"); + const timeoutMs = createProviderOperationTimeoutResolver({ + deadline: params.deadline, + defaultTimeoutMs: params.deadline.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + if (!params.allowPrivateNetwork && !params.dispatcherPolicy) { + const response = await fetchProviderDownloadResponse({ + url: params.url, + init: params.init, + deadline: params.deadline, + fetchFn: params.fetchFn, + provider: "openai", + requestFailedMessage: "OpenAI video download failed", + }); + return { + response, + release: async () => {}, + }; + } + + return await executeProviderOperationWithRetry({ + provider: "openai", + stage: "download", + operation: async () => { + const result = await fetchWithTimeoutGuarded( + params.url, + params.init, + timeoutMs(), + params.fetchFn, + { + ...(params.allowPrivateNetwork ? { ssrfPolicy: { allowPrivateNetwork: true } } : {}), + ...(params.dispatcherPolicy ? { dispatcherPolicy: params.dispatcherPolicy } : {}), + auditContext: "openai-video-download", + }, + ); + try { + await assertOkOrThrowHttpError(result.response, "OpenAI video download failed"); + return result; + } catch (error) { + await result.release(); + throw error; + } + }, + }); +} + +export function buildOpenAIVideoGenerationProvider({ + isProviderApiKeyConfigured, + getPluginConfig, +}: { + isProviderApiKeyConfigured: OpenClawPluginApi["runtime"]["modelAuth"]["isProviderApiKeyConfigured"]; + getPluginConfig?: () => Record | undefined; +}): VideoGenerationProvider { + return { + id: "openai-video", + aliases: ["openai"], + label: "OpenAI", + defaultModel: DEFAULT_OPENAI_VIDEO_MODEL, + models: [DEFAULT_OPENAI_VIDEO_MODEL, "sora-2-pro", "gemini-video"], + isConfigured: (ctx) => { + const pluginCfg = resolveVideoConfig(ctx.cfg, getPluginConfig?.()); + if ( + resolvePluginApiKey(pluginCfg?.apiKey) || + Boolean(pluginCfg?.apiKey) || + normalizeOptionalString(process.env.OPENAI_VIDEO_API_KEY) || + normalizeOptionalString(process.env.GEMINI_WEB2API_API_KEY) || + normalizeOptionalString(process.env.CCLOAD_API_KEY) + ) { + return true; + } + return isProviderApiKeyConfigured({ + provider: "openai", + ...ctx, + profileTypes: ["api_key"], + }); + }, + capabilities: { + generate: { + maxVideos: 1, + maxDurationSeconds: 12, + supportedDurationSeconds: OPENAI_VIDEO_SECONDS, + supportsSize: true, + sizes: OPENAI_VIDEO_SIZES, + }, + imageToVideo: { + enabled: true, + maxVideos: 1, + maxInputImages: 1, + maxDurationSeconds: 12, + supportedDurationSeconds: OPENAI_VIDEO_SECONDS, + supportsSize: true, + sizes: OPENAI_VIDEO_SIZES, + }, + videoToVideo: { + enabled: true, + maxVideos: 1, + maxInputVideos: 1, + }, + }, + async generateVideo(req) { + const pluginCfg = resolveVideoConfig(req.cfg, getPluginConfig?.()); + const explicitApiKey = + resolvePluginApiKey(pluginCfg?.apiKey) ?? + normalizeOptionalString(process.env.OPENAI_VIDEO_API_KEY) ?? + normalizeOptionalString(process.env.GEMINI_WEB2API_API_KEY) ?? + normalizeOptionalString(process.env.CCLOAD_API_KEY); + + let apiKey = explicitApiKey; + if (!apiKey) { + const { resolveApiKeyForProvider } = + await import("openclaw/plugin-sdk/provider-auth-runtime"); + const auth = await resolveApiKeyForProvider({ + provider: "openai", + cfg: req.cfg, + agentDir: req.agentDir, + store: req.authStore, + modelApi: "openai-responses", + }); + if (!auth.apiKey || (auth.mode !== undefined && auth.mode !== "api-key")) { + throw new Error("OpenAI API key missing"); + } + apiKey = auth.apiKey; + } + + const [ + { + assertOkOrThrowHttpError, + createProviderOperationDeadline, + createProviderOperationTimeoutResolver, + pollProviderOperationJson, + postJsonRequest, + postMultipartRequest, + readProviderJsonResponse, + resolveProviderOperationTimeoutMs, + resolveProviderHttpRequestConfig, + sanitizeConfiguredModelProviderRequest, + }, + { downloadGeneratedVideoAsset, resolveGeneratedMediaMaxBytes }, + ] = await Promise.all([ + import("openclaw/plugin-sdk/provider-http"), + import("openclaw/plugin-sdk/media-generation-runtime"), + ]); + const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "OpenAI video generation", + }); + const providerConfig = req.cfg.models?.providers?.openai; + const allowPrivate = pluginCfg?.allowPrivateNetwork ?? true; + const requestConfig = sanitizeConfiguredModelProviderRequest({ + ...providerConfig?.request, + allowPrivateNetwork: allowPrivate, + }); + const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = + resolveProviderHttpRequestConfig({ + baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg, pluginCfg), + defaultBaseUrl: DEFAULT_OPENAI_VIDEO_BASE_URL, + allowPrivateNetwork: allowPrivate, + request: requestConfig, + defaultHeaders: { + Authorization: `Bearer ${apiKey}`, + }, + provider: "openai", + capability: "video", + transport: "http", + }); + + const model = + normalizeOptionalString(req.model) ?? + normalizeOptionalString(pluginCfg?.defaultModel) ?? + DEFAULT_OPENAI_VIDEO_MODEL; + const seconds = resolveDurationSeconds(req.durationSeconds); + const size = resolveSize(req); + const referenceAsset = resolveReferenceAsset(req); + const isVideoEdit = referenceAsset?.kind === "video"; + const submitTimeoutMs = resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }); + + let response: Response; + let release: () => Promise; + + if (referenceAsset) { + const form = new FormData(); + form.set("prompt", req.prompt); + if (isVideoEdit) { + form.set("video", referenceAsset.file); + } else { + form.set("model", model); + if (seconds) { + form.set("seconds", seconds); + } + if (size) { + form.set("size", size); + } + form.set("input_reference", referenceAsset.file); + } + const multipartHeaders = new Headers(headers); + multipartHeaders.delete("Content-Type"); + ({ response, release } = await postMultipartRequest({ + url: `${baseUrl}/videos${isVideoEdit ? "/edits" : ""}`, + headers: multipartHeaders, + body: form, + timeoutMs: submitTimeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + })); + } else { + const jsonHeaders = new Headers(headers); + jsonHeaders.set("Content-Type", "application/json"); + const jsonBody: Record = { + model, + prompt: req.prompt, + ...(seconds ? { seconds } : {}), + ...(size ? { size } : {}), + }; + ({ response, release } = await postJsonRequest({ + url: `${baseUrl}/videos`, + headers: jsonHeaders, + body: jsonBody, + timeoutMs: submitTimeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + })); + } + + let submitted: OpenAIVideoResponse; + try { + await assertOkOrThrowHttpError(response, "OpenAI video generation failed"); + submitted = await readProviderJsonResponse( + response, + "OpenAI video generation failed", + ); + } finally { + await release(); + } + const failureMessage = readOpenAIVideoFailureMessage(submitted); + if (failureMessage) { + throw new Error(failureMessage); + } + const videoId = normalizeOptionalString(submitted.id); + if (!videoId) { + throw new Error("OpenAI video generation response missing video id"); + } + const completed = + submitted.status === "completed" + ? submitted + : await pollProviderOperationJson({ + url: `${baseUrl}/videos/${videoId}`, + headers, + deadline: createProviderOperationDeadline({ + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), + label: `OpenAI video generation task ${videoId}`, + }), + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + fetchFn, + maxAttempts: MAX_POLL_ATTEMPTS, + pollIntervalMs: POLL_INTERVAL_MS, + requestFailedMessage: "OpenAI video status request failed", + timeoutMessage: `OpenAI video generation task ${videoId} did not finish in time`, + allowPrivateNetwork, + dispatcherPolicy, + auditContext: "openai-video-status", + isComplete: (payload: OpenAIVideoResponse) => payload.status === "completed", + getFailureMessage: readOpenAIVideoFailureMessage, + }); + const url = new URL(`${baseUrl}/videos/${videoId}/content`); + url.searchParams.set("variant", "video"); + const video = await downloadGeneratedVideoAsset({ + url: url.toString(), + timeoutMs: createProviderOperationTimeoutResolver({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + fetchFn, + provider: "openai", + label: "OpenAI generated video download", + requestFailedMessage: "OpenAI video download failed", + maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "video"), + validateBinaryResponse: true, + fetchResponse: async ({ deadline: downloadDeadline }: { deadline: any }) => + await fetchOpenAIVideoDownload({ + url: url.toString(), + init: { + method: "GET", + headers: new Headers({ + ...Object.fromEntries(headers.entries()), + Accept: "application/binary", + }), + }, + deadline: downloadDeadline, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + }), + }); + return { + videos: [video], + model: completed.model ?? submitted.model ?? model, + metadata: { + videoId, + status: completed.status, + seconds: completed.seconds ?? submitted.seconds, + size: completed.size ?? submitted.size, + }, + }; + }, + }; +} diff --git a/tsconfig.json b/tsconfig.json index eb6660e..b483562 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,6 +10,9 @@ "skipLibCheck": true }, "include": [ - "src/index.ts" + "src/**/*" + ], + "exclude": [ + "src/**/*.test.ts" ] }