From 1abcde7be933ae85fbe79366907ecc4fa90e5915 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:17:18 -0700 Subject: [PATCH 01/10] fix(media): validate upload paths before generation Allow media tools to consume chat-scoped uploads directly while keeping outputs under files/. Fail malformed or unresolved declarations before provider work and regenerate the Mothership-derived tool schemas. --- .../lib/copilot/generated/tool-catalog-v1.ts | 275 +++------------- .../lib/copilot/generated/tool-schemas-v1.ts | 299 +++--------------- .../tools/handlers/upload-file-reader.ts | 11 + .../tools/server/image/generate-image.test.ts | 211 ++++++++++++ .../tools/server/image/generate-image.ts | 68 ++-- .../lib/copilot/tools/server/media/ffmpeg.ts | 25 +- .../tools/server/media/file-paths.test.ts | 158 +++++++++ .../copilot/tools/server/media/file-paths.ts | 109 +++++++ .../tools/server/media/generate-audio.ts | 44 ++- .../tools/server/media/generate-video.ts | 34 +- .../media/media-tools-validation.test.ts | 176 +++++++++++ 11 files changed, 859 insertions(+), 551 deletions(-) create mode 100644 apps/sim/lib/copilot/tools/server/image/generate-image.test.ts create mode 100644 apps/sim/lib/copilot/tools/server/media/file-paths.test.ts create mode 100644 apps/sim/lib/copilot/tools/server/media/file-paths.ts create mode 100644 apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 77fa9e44091..843cf5c68ff 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -1243,66 +1243,27 @@ export const Ffmpeg: ToolCatalogEntry = { height: { type: 'number', description: 'Target height in pixels (scale_pad).' }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: + 'One or more media inputs from files/... or uploads/..., in operation order.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, loopToVideo: { type: 'boolean', @@ -1333,20 +1294,15 @@ export const Ffmpeg: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1357,19 +1313,18 @@ export const Ffmpeg: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, position: { type: 'string', @@ -1591,66 +1546,27 @@ export const GenerateAudio: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one reference voice sample from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, instrumental: { type: 'boolean', @@ -1670,20 +1586,15 @@ export const GenerateAudio: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1694,19 +1605,18 @@ export const GenerateAudio: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1741,84 +1651,39 @@ export const GenerateImage: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'One or more reference images to edit, combine, or restyle.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1829,19 +1694,18 @@ export const GenerateImage: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1879,66 +1743,27 @@ export const GenerateVideo: ToolCatalogEntry = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one start-frame image from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { type: 'string', description: 'Canonical VFS table path when available.' }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { type: 'string', description: 'Workspace table ID.' }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, model: { type: 'string', @@ -1964,20 +1789,15 @@ export const GenerateVideo: ToolCatalogEntry = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1988,19 +1808,18 @@ export const GenerateVideo: ToolCatalogEntry = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index 00646f08622..ba98ea7248e 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -1058,72 +1058,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: + 'One or more media inputs from files/... or uploads/..., in operation order.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, loopToVideo: { type: 'boolean', @@ -1154,20 +1109,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1178,19 +1128,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, position: { type: 'string', @@ -1405,72 +1354,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one reference voice sample from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, instrumental: { type: 'boolean', @@ -1490,20 +1394,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1514,19 +1413,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1558,90 +1456,39 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'One or more reference images to edit, combine, or restyle.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, }, }, + required: ['files'], }, outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1652,19 +1499,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', @@ -1696,72 +1542,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: - 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', + description: 'Existing media files to use as generation or composition inputs.', properties: { - directories: { - type: 'array', - description: - 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: - 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', - }, - }, - required: ['path'], - }, - }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Exactly one start-frame image from files/... or uploads/....', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', - }, - sandboxPath: { - type: 'string', - description: - 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', + 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', }, }, required: ['path'], }, - }, - tables: { - type: 'array', - description: 'Workspace tables to mount as CSV files.', - items: { - type: 'object', - properties: { - path: { - type: 'string', - description: 'Canonical VFS table path when available.', - }, - sandboxPath: { - type: 'string', - description: 'Optional full sandbox path for the mounted CSV.', - }, - tableId: { - type: 'string', - description: 'Workspace table ID.', - }, - }, - }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, model: { type: 'string', @@ -1787,20 +1588,15 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'Workspace files to create or overwrite from returned code results or sandbox-created files.', + 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', properties: { files: { type: 'array', description: - 'File outputs. Missing parent folders are created automatically for create mode.', + 'Exactly one file output. Parent folders must already exist for create mode.', items: { type: 'object', properties: { - format: { - type: 'string', - description: 'Optional serialization format for returned values.', - enum: ['json', 'csv', 'txt', 'md', 'html'], - }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1811,19 +1607,18 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['create', 'overwrite'], }, path: { - type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', - }, - sandboxPath: { type: 'string', description: - 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', + 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', }, }, required: ['path', 'mode'], }, + minItems: 1, + maxItems: 1, }, }, + required: ['files'], }, prompt: { type: 'string', diff --git a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts index 171a1589a25..95a391eeb52 100644 --- a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts +++ b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts @@ -96,6 +96,17 @@ function toWorkspaceFileRecord(row: typeof workspaceFiles.$inferSelect): Workspa } } +/** + * Resolve a chat-scoped upload to the same file-record shape used by workspace files. + */ +export async function resolveChatUpload( + filename: string, + chatId: string +): Promise { + const row = await findMothershipUploadRowByChatAndName(chatId, filename) + return row ? toWorkspaceFileRecord(row) : null +} + /** * Resolve a mothership upload row by VFS name (the collision-disambiguated `displayName` * for new rows, or `originalName` for legacy rows that predate the column). Prefers an diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts new file mode 100644 index 00000000000..2082006e735 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts @@ -0,0 +1,211 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + fetchWorkspaceFileBuffer: vi.fn(), + generateContent: vi.fn(), + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), + writeWorkspaceFileByPath: vi.fn(), +})) + +vi.mock('@google/genai', () => ({ + GoogleGenAI: class { + models = { generateContent: mocks.generateContent } + }, +})) + +vi.mock('@/lib/core/config/api-keys', () => ({ + getRotatingApiKey: vi.fn(() => 'test-api-key'), +})) + +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchWorkspaceFileBuffer: mocks.fetchWorkspaceFileBuffer, + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) + +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, + writeWorkspaceFileByPath: mocks.writeWorkspaceFileByPath, +})) + +import { generateImageServerTool } from '@/lib/copilot/tools/server/image/generate-image' + +const CONTEXT = { + userId: 'user-1', + workspaceId: 'workspace-1', + chatId: 'chat-1', +} + +const IMAGE_RECORD = { + id: 'wf_image', + workspaceId: 'workspace-1', + name: 'person.png', + key: 'uploads/person.png', + path: '/api/files/serve/uploads%2Fperson.png', + size: 3, + type: 'image/png', + uploadedBy: 'user-1', + uploadedAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), + storageContext: 'mothership' as const, +} + +describe('generateImageServerTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('image')) + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + mocks.generateContent.mockResolvedValue({ + candidates: [ + { content: { parts: [{ inlineData: { data: 'aW1hZ2U=', mimeType: 'image/png' } }] } }, + ], + }) + mocks.writeWorkspaceFileByPath.mockResolvedValue({ + id: 'wf_result', + name: 'result.png', + vfsPath: 'files/result.png', + mode: 'create', + downloadUrl: '/download/result.png', + }) + }) + + it('loads a chat upload when inputs.files uses an uploads/ path', async () => { + mocks.resolveChatUpload.mockResolvedValue(IMAGE_RECORD) + + const result = await generateImageServerTool.execute( + { + prompt: 'Preserve this person and put them in a library', + inputs: { files: [{ path: 'uploads/person.png' }] }, + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('person.png', 'chat-1') + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + expect(mocks.fetchWorkspaceFileBuffer).toHaveBeenCalledWith(IMAGE_RECORD) + }) + + it('fails before generation when an input path cannot be resolved', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + const result = await generateImageServerTool.execute( + { + prompt: 'Edit the reference image', + inputs: { files: [{ path: 'files/missing.png' }] }, + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: 'Failed to generate image: Input file not found: files/missing.png', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('fails before generation when outputs.files targets uploads/', async () => { + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: 'uploads/result.png' }] }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: + 'Failed to generate image: Media output paths must start with "files/"; uploads/ paths are read-only: uploads/result.png', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('fails before generation when an explicit output path is empty', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('Workspace file paths must start with "files/"') + ) + + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: '' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects additional output declarations before generation', async () => { + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { + files: [{ path: 'files/one.png' }, { path: 'uploads/two.png' }], + }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: 'Failed to generate image: Output requires exactly one file; received 2', + }) + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mocks.generateContent).not.toHaveBeenCalled() + }) + + it('rejects explicitly empty input and output containers before generation', async () => { + const emptyInputs = await generateImageServerTool.execute( + { prompt: 'Edit the supplied image', inputs: {} }, + CONTEXT + ) + const emptyOutputs = await generateImageServerTool.execute( + { prompt: 'Create a portrait', outputs: {} }, + CONTEXT + ) + + expect(emptyInputs).toEqual({ + success: false, + message: 'Failed to generate image: Input requires at least one file', + }) + expect(emptyOutputs).toEqual({ + success: false, + message: 'Failed to generate image: Output requires exactly one file; received 0', + }) + expect(mocks.generateContent).not.toHaveBeenCalled() + }) + + it('fails before generation when canonical output preflight rejects a files/ target', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('File already exists at files/result.png') + ) + + const result = await generateImageServerTool.execute( + { + prompt: 'Create a portrait', + outputs: { files: [{ path: 'files/result.png' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateContent).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.ts index 3a6c25b8240..b6dc6205c76 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.ts @@ -1,18 +1,20 @@ import { GoogleGenAI, type Part } from '@google/genai' import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { GenerateImage } from '@/lib/copilot/generated/tool-catalog-v1' import { assertServerToolNotAborted, type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' +import { + prepareMediaOutput, + requireMediaFileDeclarations, + resolveMediaInputFile, +} from '@/lib/copilot/tools/server/media/file-paths' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' import { getRotatingApiKey } from '@/lib/core/config/api-keys' -import { - fetchWorkspaceFileBuffer, - resolveWorkspaceFileReference, -} from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager' const logger = createLogger('GenerateImageTool') @@ -74,6 +76,12 @@ export const generateImageServerTool: BaseServerTool file.path) ?? [] + const referencePaths = params.inputs + ? requireMediaFileDeclarations(params.inputs.files, 'Input').map((file) => file.path) + : [] if (referencePaths.length) { for (const filePath of referencePaths) { - try { - const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath) - if (fileRecord) { - const buffer = await fetchWorkspaceFileBuffer(fileRecord) - const base64 = buffer.toString('base64') - const mime = fileRecord.type || 'image/png' - parts.push({ - inlineData: { mimeType: mime, data: base64 }, - }) - logger.info('Loaded reference image', { - filePath, - name: fileRecord.name, - size: buffer.length, - mimeType: mime, - }) - } else { - logger.warn('Reference file not found, skipping', { filePath }) - } - } catch (err) { - logger.warn('Failed to load reference image, skipping', { - filePath, - error: toError(err).message, - }) - } + const fileRecord = await resolveMediaInputFile({ + workspaceId, + chatId: context.chatId, + path: filePath, + }) + const buffer = await fetchWorkspaceFileBuffer(fileRecord) + const base64 = buffer.toString('base64') + const mime = fileRecord.type || 'image/png' + parts.push({ + inlineData: { mimeType: mime, data: base64 }, + }) + logger.info('Loaded reference image', { + filePath, + name: fileRecord.name, + size: buffer.length, + mimeType: mime, + }) } } @@ -161,8 +164,7 @@ export const generateImageServerTool: BaseServerTool = { } try { + const outputFile = await prepareMediaOutput({ + output: params.outputs, + workspaceId, + userId: context.userId, + }) + const mediaFiles: MediaFile[] = [] for (const filePath of inputPaths) { - const fileRecord = await resolveWorkspaceFileReference(workspaceId, filePath) - if (!fileRecord) { - return { success: false, message: `Input file not found: ${filePath}` } - } + const fileRecord = await resolveMediaInputFile({ + workspaceId, + chatId: context.chatId, + path: filePath, + }) const buffer = await fetchWorkspaceFileBuffer(fileRecord) mediaFiles.push({ buffer, @@ -123,7 +131,6 @@ export const ffmpegServerTool: BaseServerTool = { return { success: false, message: `ffmpeg ${params.operation} produced no output` } } - const outputFile = params.outputs?.files?.[0] const outputPath = outputFile?.path || `files/ffmpeg-${params.operation}.${result.ext}` const mode = outputFile?.mode ?? 'create' diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts new file mode 100644 index 00000000000..45d3253ca03 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), +})) + +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) + +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, +})) + +import { + getSingleMediaFileDeclaration, + resolveMediaInputFile, + validateMediaOutputFile, +} from '@/lib/copilot/tools/server/media/file-paths' + +const FILE_RECORD = { id: 'wf_file', name: 'portrait.png' } + +describe('media file paths', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + }) + + it('resolves uploads/ inputs within the current chat', async () => { + mocks.resolveChatUpload.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'uploads/My%20Portrait.png', + }) + ).resolves.toBe(FILE_RECORD) + + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('My%20Portrait.png', 'chat-1') + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + }) + + it('rejects unresolved inputs instead of dropping them', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'files/missing.png', + }) + ).rejects.toThrow('Input file not found: files/missing.png') + }) + + it('preserves legacy workspace IDs and plain-name references', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'portrait.png', + }) + ).resolves.toBe(FILE_RECORD) + + await resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'wf_file', + }) + + expect(mocks.resolveWorkspaceFileReference).toHaveBeenNthCalledWith( + 1, + 'workspace-1', + 'portrait.png' + ) + expect(mocks.resolveWorkspaceFileReference).toHaveBeenNthCalledWith(2, 'workspace-1', 'wf_file') + }) + + it('preflights outputs through the canonical workspace writer policy', async () => { + await validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'files/Campaign/portrait.png', + mode: 'overwrite', + mimeType: 'image/png', + }) + + expect(mocks.validateWorkspaceFileWriteTarget).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + userId: 'user-1', + target: { + path: 'files/Campaign/portrait.png', + mode: 'overwrite', + mimeType: 'image/png', + }, + }) + }) + + it('rejects uploads/ output paths before canonical resolution', async () => { + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'uploads/portrait.png', + mode: 'create', + }) + ).rejects.toThrow('Media output paths must start with "files/"') + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) + + it('propagates canonical writer validation errors for files/ paths', async () => { + mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( + new Error('File already exists at files/portrait.png') + ) + + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'files/portrait.png', + mode: 'create', + }) + ).rejects.toThrow('File already exists at files/portrait.png') + expect(mocks.validateWorkspaceFileWriteTarget).toHaveBeenCalledOnce() + }) + + it('rejects non-file output namespaces before canonical resolution', async () => { + await expect( + validateMediaOutputFile({ + workspaceId: 'workspace-1', + userId: 'user-1', + path: 'workflows/My%20Flow/state.json', + mode: 'overwrite', + }) + ).rejects.toThrow('Media output paths must start with "files/"') + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) + + it('rejects extra declarations for single-file media parameters', () => { + expect(() => + getSingleMediaFileDeclaration([{ path: 'files/a.png' }, { path: 'files/b.png' }], 'Output') + ).toThrow('Output requires exactly one file; received 2') + expect(() => getSingleMediaFileDeclaration([], 'Output')).toThrow( + 'Output requires exactly one file; received 0' + ) + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.ts new file mode 100644 index 00000000000..69b661aedc1 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.ts @@ -0,0 +1,109 @@ +import { resolveChatUpload } from '@/lib/copilot/tools/handlers/upload-file-reader' +import { + validateWorkspaceFileWriteTarget, + type WorkspaceFileWriteMode, +} from '@/lib/copilot/vfs/resource-writer' +import { + resolveWorkspaceFileReference, + type WorkspaceFileRecord, +} from '@/lib/uploads/contexts/workspace/workspace-file-manager' + +const UPLOADS_PREFIX = 'uploads/' +const FILES_PREFIX = 'files/' + +interface MediaOutputDeclaration { + path: string + mode?: WorkspaceFileWriteMode + mimeType?: string +} + +export function getSingleMediaFileDeclaration( + files: T[] | undefined, + label: 'Input' | 'Output' +): T { + if (files?.length !== 1) { + throw new Error(`${label} requires exactly one file; received ${files?.length ?? 0}`) + } + return files[0] +} + +export function requireMediaFileDeclarations( + files: T[] | undefined, + label: 'Input' | 'Output' +): T[] { + if (!files?.length) { + throw new Error(`${label} requires at least one file`) + } + return files +} + +/** + * Resolve a media input from either persistent workspace files or chat-scoped uploads. + */ +export async function resolveMediaInputFile(args: { + workspaceId: string + chatId?: string + path: string +}): Promise { + const path = args.path.trim() + let file: WorkspaceFileRecord | null + + if (path.startsWith(UPLOADS_PREFIX)) { + const filename = path.slice(UPLOADS_PREFIX.length) + if (!filename || filename.includes('/')) { + throw new Error(`Upload input path must identify a file: ${args.path}`) + } + if (!args.chatId) { + throw new Error(`Chat context is required for upload input: ${path}`) + } + file = await resolveChatUpload(filename, args.chatId) + } else { + file = await resolveWorkspaceFileReference(args.workspaceId, path) + } + + if (!file) { + throw new Error(`Input file not found: ${args.path}`) + } + return file +} + +/** + * Preflight a user-supplied media output through the canonical workspace writer policy. + */ +export async function validateMediaOutputFile(args: { + workspaceId: string + userId: string + path: string + mode: WorkspaceFileWriteMode + mimeType?: string +}): Promise { + if (!args.path.trim().startsWith(FILES_PREFIX)) { + throw new Error( + `Media output paths must start with "files/"; uploads/ paths are read-only: ${args.path}` + ) + } + await validateWorkspaceFileWriteTarget({ + workspaceId: args.workspaceId, + userId: args.userId, + target: { path: args.path, mode: args.mode, mimeType: args.mimeType }, + }) +} + +export async function prepareMediaOutput(args: { + output?: { files?: T[] } + workspaceId: string + userId: string +}): Promise<(T & { mode: WorkspaceFileWriteMode }) | undefined> { + if (!args.output) return undefined + + const file = getSingleMediaFileDeclaration(args.output.files, 'Output') + const output = { ...file, mode: file.mode ?? 'create' } + await validateMediaOutputFile({ + workspaceId: args.workspaceId, + userId: args.userId, + path: output.path, + mode: output.mode, + mimeType: output.mimeType, + }) + return output +} diff --git a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts index 6d224686397..c3f430137c3 100644 --- a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts +++ b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts @@ -6,12 +6,14 @@ import { type BaseServerTool, type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' +import { + getSingleMediaFileDeclaration, + prepareMediaOutput, + resolveMediaInputFile, +} from '@/lib/copilot/tools/server/media/file-paths' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' import { type AudioType, generateFalAudio } from '@/lib/media/falai-audio' -import { - fetchWorkspaceFileBuffer, - resolveWorkspaceFileReference, -} from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager' const logger = createLogger('GenerateAudioTool') @@ -80,20 +82,29 @@ export const generateAudioServerTool: BaseServerTool ({ + fetchWorkspaceFileBuffer: vi.fn(), + generateFalAudio: vi.fn(), + generateFalVideo: vi.fn(), + resolveChatUpload: vi.fn(), + resolveWorkspaceFileReference: vi.fn(), + runFfmpegOperation: vi.fn(), + validateWorkspaceFileWriteTarget: vi.fn(), + writeWorkspaceFileByPath: vi.fn(), +})) + +vi.mock('@/lib/media/falai-audio', () => ({ generateFalAudio: mocks.generateFalAudio })) +vi.mock('@/lib/media/falai-video', () => ({ generateFalVideo: mocks.generateFalVideo })) +vi.mock('@/lib/media/ffmpeg', () => ({ runFfmpegOperation: mocks.runFfmpegOperation })) +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mocks.resolveChatUpload, +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchWorkspaceFileBuffer: mocks.fetchWorkspaceFileBuffer, + resolveWorkspaceFileReference: mocks.resolveWorkspaceFileReference, +})) +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mocks.validateWorkspaceFileWriteTarget, + writeWorkspaceFileByPath: mocks.writeWorkspaceFileByPath, +})) + +import { ffmpegServerTool } from '@/lib/copilot/tools/server/media/ffmpeg' +import { generateAudioServerTool } from '@/lib/copilot/tools/server/media/generate-audio' +import { generateVideoServerTool } from '@/lib/copilot/tools/server/media/generate-video' + +const CONTEXT = { userId: 'user-1', workspaceId: 'workspace-1', chatId: 'chat-1' } +const UPLOAD_RECORD = { + id: 'wf_upload', + name: 'start.png', + type: 'image/png', + storageContext: 'mothership', +} + +describe('media tool path validation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateWorkspaceFileWriteTarget.mockResolvedValue({}) + mocks.fetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('media')) + mocks.generateFalVideo.mockResolvedValue({ + buffer: Buffer.from('video'), + contentType: 'video/mp4', + model: 'veo-3.1-fast', + cost: { costDollars: 0.1 }, + }) + mocks.generateFalAudio.mockResolvedValue({ + buffer: Buffer.from('audio'), + contentType: 'audio/mpeg', + model: 'fal-ai/f5-tts', + cost: { costDollars: 0.05 }, + }) + mocks.runFfmpegOperation.mockResolvedValue({ + buffer: Buffer.from('converted'), + contentType: 'video/mp4', + ext: 'mp4', + }) + mocks.writeWorkspaceFileByPath.mockResolvedValue({ + id: 'wf_output', + name: 'output.mp4', + vfsPath: 'files/output.mp4', + mode: 'create', + }) + }) + + it('passes uploads/ start frames to video generation', async () => { + mocks.resolveChatUpload.mockResolvedValue(UPLOAD_RECORD) + + const result = await generateVideoServerTool.execute( + { + prompt: 'Animate this portrait', + inputs: { files: [{ path: 'uploads/start.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('start.png', 'chat-1') + expect(mocks.generateFalVideo).toHaveBeenCalledWith( + expect.objectContaining({ imageDataUri: 'data:image/png;base64,bWVkaWE=' }) + ) + }) + + it('rejects empty audio reference paths before calling the provider', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(null) + + const result = await generateAudioServerTool.execute( + { prompt: 'Clone this voice', inputs: { files: [{ path: '' }] } }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalAudio).not.toHaveBeenCalled() + }) + + it('passes uploads/ voice samples to audio generation', async () => { + mocks.resolveChatUpload.mockResolvedValue({ + ...UPLOAD_RECORD, + name: 'voice.wav', + type: 'audio/wav', + }) + + const result = await generateAudioServerTool.execute( + { + prompt: 'Read this line in the supplied voice', + inputs: { files: [{ path: 'uploads/voice.wav' }] }, + outputs: { files: [{ path: 'files/voice.mp3' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('voice.wav', 'chat-1') + expect(mocks.generateFalAudio).toHaveBeenCalledWith( + expect.objectContaining({ voiceSampleDataUri: 'data:audio/wav;base64,bWVkaWE=' }) + ) + }) + + it('rejects invalid ffmpeg output paths before processing inputs', async () => { + const result = await ffmpegServerTool.execute( + { + operation: 'convert', + inputs: { files: [{ path: 'files/input.mov' }] }, + outputs: { files: [{ path: 'uploads/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() + expect(mocks.runFfmpegOperation).not.toHaveBeenCalled() + }) + + it('passes uploads/ media files to ffmpeg', async () => { + mocks.resolveChatUpload.mockResolvedValue(UPLOAD_RECORD) + + const result = await ffmpegServerTool.execute( + { + operation: 'convert', + inputs: { files: [{ path: 'uploads/start.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('start.png', 'chat-1') + expect(mocks.runFfmpegOperation).toHaveBeenCalledWith( + 'convert', + [expect.objectContaining({ buffer: Buffer.from('media'), name: 'start.png' })], + expect.any(Object) + ) + }) + + it('rejects extra video inputs before calling the provider', async () => { + const result = await generateVideoServerTool.execute( + { + prompt: 'Animate these portraits', + inputs: { files: [{ path: 'files/one.png' }, { path: 'files/two.png' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalVideo).not.toHaveBeenCalled() + }) +}) From ba6fe69743c7a7f66e999a23d156cfe0178ff212 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:42:32 -0700 Subject: [PATCH 02/10] fix(media): preflight all writable outputs --- .../lib/copilot/generated/tool-catalog-v1.ts | 6 ++-- .../lib/copilot/generated/tool-schemas-v1.ts | 6 ++-- .../tools/server/image/generate-image.test.ts | 6 +++- .../tools/server/image/generate-image.ts | 7 ++-- .../lib/copilot/tools/server/media/ffmpeg.ts | 24 ++++++++----- .../tools/server/media/file-paths.test.ts | 26 ++++++++++++++ .../copilot/tools/server/media/file-paths.ts | 12 +++---- .../tools/server/media/generate-audio.ts | 17 ++------- .../tools/server/media/generate-video.ts | 6 ++-- .../media/media-tools-validation.test.ts | 35 ++++++++++++++++++- 10 files changed, 100 insertions(+), 45 deletions(-) diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 843cf5c68ff..20ed170b981 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -1630,7 +1630,7 @@ export const GenerateAudio: ToolCatalogEntry = { }, voice: { type: 'string', description: 'Optional voice name or id for speech.' }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], @@ -1713,7 +1713,7 @@ export const GenerateImage: ToolCatalogEntry = { 'Detailed text description of the image to generate, or editing instructions when editing the image(s) passed in `inputs.files`.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], @@ -1836,7 +1836,7 @@ export const GenerateVideo: ToolCatalogEntry = { enum: ['720p', '1080p', '4k'], }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index ba98ea7248e..15d51724314 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -1441,7 +1441,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { description: 'Optional voice name or id for speech.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, @@ -1518,7 +1518,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'Detailed text description of the image to generate, or editing instructions when editing the image(s) passed in `inputs.files`.', }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, @@ -1635,7 +1635,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['720p', '1080p', '4k'], }, }, - required: ['prompt'], + required: ['prompt', 'outputs'], }, resultSchema: undefined, }, diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts index 2082006e735..d5e7e5aed8a 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts @@ -172,7 +172,11 @@ describe('generateImageServerTool', () => { it('rejects explicitly empty input and output containers before generation', async () => { const emptyInputs = await generateImageServerTool.execute( - { prompt: 'Edit the supplied image', inputs: {} }, + { + prompt: 'Edit the supplied image', + inputs: {}, + outputs: { files: [{ path: 'files/result.png' }] }, + }, CONTEXT ) const emptyOutputs = await generateImageServerTool.execute( diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.ts index b6dc6205c76..0cda9700a22 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.ts @@ -163,10 +163,9 @@ export const generateImageServerTool: BaseServerTool = { } try { - const outputFile = await prepareMediaOutput({ - output: params.outputs, - workspaceId, - userId: context.userId, - }) + const outputFile = + params.operation === 'probe' + ? undefined + : await prepareMediaOutput({ + output: params.outputs, + workspaceId, + userId: context.userId, + }) const mediaFiles: MediaFile[] = [] for (const filePath of inputPaths) { @@ -127,18 +130,21 @@ export const ffmpegServerTool: BaseServerTool = { } } - if (!result.buffer || !result.ext) { + if (!result.buffer) { return { success: false, message: `ffmpeg ${params.operation} produced no output` } } + if (!outputFile) { + return { success: false, message: `ffmpeg ${params.operation} requires an output file` } + } - const outputPath = outputFile?.path || `files/ffmpeg-${params.operation}.${result.ext}` - const mode = outputFile?.mode ?? 'create' + const outputPath = outputFile.path + const mode = outputFile.mode assertServerToolNotAborted(context) const written = await writeWorkspaceFileByPath({ workspaceId, userId: context.userId, - target: { path: outputPath, mode, mimeType: outputFile?.mimeType }, + target: { path: outputPath, mode, mimeType: outputFile.mimeType }, buffer: result.buffer, inferredMimeType: result.contentType || 'application/octet-stream', }) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts index 45d3253ca03..f757e960f0a 100644 --- a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts @@ -23,6 +23,7 @@ vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ import { getSingleMediaFileDeclaration, + prepareMediaOutput, resolveMediaInputFile, validateMediaOutputFile, } from '@/lib/copilot/tools/server/media/file-paths' @@ -50,6 +51,20 @@ describe('media file paths', () => { expect(mocks.resolveWorkspaceFileReference).not.toHaveBeenCalled() }) + it('resolves uploads/ inputs with the read-compatible /content suffix', async () => { + mocks.resolveChatUpload.mockResolvedValue(FILE_RECORD) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'uploads/My%20Portrait.png/content', + }) + ).resolves.toBe(FILE_RECORD) + + expect(mocks.resolveChatUpload).toHaveBeenCalledWith('My%20Portrait.png', 'chat-1') + }) + it('rejects unresolved inputs instead of dropping them', async () => { mocks.resolveWorkspaceFileReference.mockResolvedValue(null) @@ -155,4 +170,15 @@ describe('media file paths', () => { 'Output requires exactly one file; received 0' ) }) + + it('requires an explicit output before provider work can begin', async () => { + await expect( + prepareMediaOutput({ + workspaceId: 'workspace-1', + userId: 'user-1', + }) + ).rejects.toThrow('Output requires exactly one file; received 0') + + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.ts index 69b661aedc1..ec07aab70e6 100644 --- a/apps/sim/lib/copilot/tools/server/media/file-paths.ts +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.ts @@ -49,8 +49,8 @@ export async function resolveMediaInputFile(args: { let file: WorkspaceFileRecord | null if (path.startsWith(UPLOADS_PREFIX)) { - const filename = path.slice(UPLOADS_PREFIX.length) - if (!filename || filename.includes('/')) { + const filename = path.slice(UPLOADS_PREFIX.length).split('/')[0] + if (!filename) { throw new Error(`Upload input path must identify a file: ${args.path}`) } if (!args.chatId) { @@ -68,7 +68,7 @@ export async function resolveMediaInputFile(args: { } /** - * Preflight a user-supplied media output through the canonical workspace writer policy. + * Preflight a media output through the canonical workspace writer policy. */ export async function validateMediaOutputFile(args: { workspaceId: string @@ -93,10 +93,8 @@ export async function prepareMediaOutput(args: output?: { files?: T[] } workspaceId: string userId: string -}): Promise<(T & { mode: WorkspaceFileWriteMode }) | undefined> { - if (!args.output) return undefined - - const file = getSingleMediaFileDeclaration(args.output.files, 'Output') +}): Promise { + const file = getSingleMediaFileDeclaration(args.output?.files, 'Output') const output = { ...file, mode: file.mode ?? 'create' } await validateMediaOutputFile({ workspaceId: args.workspaceId, diff --git a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts index c3f430137c3..5b000d67c16 100644 --- a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts +++ b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts @@ -46,16 +46,6 @@ interface GenerateAudioResult { _serviceCost?: { service: string; cost: number } } -function audioExtFromContentType(contentType: string): string { - if (contentType.includes('wav')) return 'wav' - if (contentType.includes('mp4') || contentType.includes('m4a')) return 'm4a' - if (contentType.includes('ogg')) return 'ogg' - if (contentType.includes('flac')) return 'flac' - if (contentType.includes('aac')) return 'aac' - if (contentType.includes('opus')) return 'opus' - return 'mp3' -} - export const generateAudioServerTool: BaseServerTool = { name: GenerateAudio.id, @@ -123,15 +113,14 @@ export const generateAudioServerTool: BaseServerTool { mocks.resolveWorkspaceFileReference.mockResolvedValue(null) const result = await generateAudioServerTool.execute( - { prompt: 'Clone this voice', inputs: { files: [{ path: '' }] } }, + { + prompt: 'Clone this voice', + inputs: { files: [{ path: '' }] }, + outputs: { files: [{ path: 'files/voice.mp3' }] }, + }, CONTEXT ) @@ -140,6 +144,34 @@ describe('media tool path validation', () => { expect(mocks.runFfmpegOperation).not.toHaveBeenCalled() }) + it('ignores unused output declarations for ffmpeg probes', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue(UPLOAD_RECORD) + mocks.runFfmpegOperation.mockResolvedValue({ probe: { duration: 3 } }) + + const result = await ffmpegServerTool.execute( + { + operation: 'probe', + inputs: { files: [{ path: 'files/input.mov' }] }, + outputs: { files: [{ path: 'uploads/unused.json' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects missing generated-media outputs before calling the provider', async () => { + const result = await generateVideoServerTool.execute( + { prompt: 'Create a launch video' }, + CONTEXT + ) + + expect(result.success).toBe(false) + expect(mocks.generateFalVideo).not.toHaveBeenCalled() + }) + it('passes uploads/ media files to ffmpeg', async () => { mocks.resolveChatUpload.mockResolvedValue(UPLOAD_RECORD) @@ -166,6 +198,7 @@ describe('media tool path validation', () => { { prompt: 'Animate these portraits', inputs: { files: [{ path: 'files/one.png' }, { path: 'files/two.png' }] }, + outputs: { files: [{ path: 'files/output.mp4' }] }, }, CONTEXT ) From fe6609957a788092187a48e8e8fe8bbf8abbd205 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:18:17 -0700 Subject: [PATCH 03/10] fix(copilot): mount chat uploads in code sandboxes Resolve uploads through the current chat, enforce workspace ownership, and reuse the existing URL or buffered sandbox mount path so function_execute and run_code can process uploaded files directly. --- .../tools/handlers/function-execute.test.ts | 58 +++++++++++++++++++ .../tools/handlers/function-execute.ts | 53 ++++++++++++----- 2 files changed, 97 insertions(+), 14 deletions(-) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts index f6494b14aa0..ec5e9a1d47e 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts @@ -18,6 +18,7 @@ const { mockFetchWorkspaceFileBuffer, mockGetSandboxWorkspaceFilePath, mockListWorkspaceFileFolders, + mockResolveChatUpload, } = vi.hoisted(() => ({ mockIsFeatureEnabled: vi.fn(), mockGetTableById: vi.fn(), @@ -33,6 +34,7 @@ const { mockFetchWorkspaceFileBuffer: vi.fn(), mockGetSandboxWorkspaceFilePath: vi.fn(), mockListWorkspaceFileFolders: vi.fn(), + mockResolveChatUpload: vi.fn(), })) vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: mockIsFeatureEnabled })) @@ -71,6 +73,9 @@ vi.mock('@/lib/copilot/vfs/workflow-aliases', () => ({ isPlanAliasPath: () => false, workflowAliasSandboxPath: (p: string) => p, })) +vi.mock('@/lib/copilot/tools/handlers/upload-file-reader', () => ({ + resolveChatUpload: mockResolveChatUpload, +})) import { executeFunctionExecute } from '@/lib/copilot/tools/handlers/function-execute' @@ -268,6 +273,14 @@ const fileRecord = { storageContext: 'workspace' as const, } +const uploadRecord = { + ...fileRecord, + id: 'upload_1', + name: 'My Report.csv', + key: 'mothership/chat_1/my-report.csv', + storageContext: 'mothership' as const, +} + describe('executeFunctionExecute file mounts', () => { beforeEach(() => { vi.clearAllMocks() @@ -309,6 +322,51 @@ describe('executeFunctionExecute file mounts', () => { expect(file.type).toBeUndefined() }) + it('mounts a chat upload at its canonical sandbox path', async () => { + mockResolveChatUpload.mockResolvedValue(uploadRecord) + + await executeFunctionExecute({ inputs: { files: [{ path: 'uploads/My%20Report.csv' }] } }, { + ...context, + chatId: 'chat_1', + } as never) + + expect(mockResolveChatUpload).toHaveBeenCalledWith('My%20Report.csv', 'chat_1') + expect(mockListWorkspaceFiles).not.toHaveBeenCalled() + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith( + 'mothership/chat_1/my-report.csv', + 'mothership', + expect.any(Number) + ) + expect(mountedFiles()[0]).toEqual({ + type: 'url', + path: '/home/user/uploads/My%20Report.csv', + url: 'https://s3.example/file?sig=abc', + }) + }) + + it('reports when a chat upload is no longer available', async () => { + mockResolveChatUpload.mockResolvedValue(null) + + await expect( + executeFunctionExecute({ inputFiles: ['uploads/missing.csv'] }, { + ...context, + chatId: 'chat_1', + } as never) + ).rejects.toThrow('Upload not found: "uploads/missing.csv"') + }) + + it('rejects a chat upload from another workspace', async () => { + mockResolveChatUpload.mockResolvedValue({ ...uploadRecord, workspaceId: 'ws_2' }) + + await expect( + executeFunctionExecute({ inputFiles: ['uploads/My%20Report.csv'] }, { + ...context, + chatId: 'chat_1', + } as never) + ).rejects.toThrow('Upload does not belong to the current workspace') + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + }) + it('cloud storage: throws when a file exceeds the per-file URL mount limit', async () => { mockFindWorkspaceFileRecord.mockReturnValue({ ...fileRecord, size: 600 * 1024 * 1024 }) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index bc32699d66a..010a2d24791 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { resolveChatUpload } from '@/lib/copilot/tools/handlers/upload-file-reader' import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils' import { resolveWorkflowAliasForWorkspace } from '@/lib/copilot/vfs/workflow-alias-resolver' import { isPlanAliasPath, workflowAliasSandboxPath } from '@/lib/copilot/vfs/workflow-aliases' @@ -29,6 +30,7 @@ const logger = createLogger('CopilotFunctionExecute') const MAX_FILE_SIZE = 10 * 1024 * 1024 const MAX_TOTAL_SIZE = 50 * 1024 * 1024 const MAX_MOUNTED_FILES = 500 +const UPLOADS_PREFIX = 'uploads/' /** * Below this row count a table mounts via the direct inline CSV path — the version-keyed snapshot @@ -170,7 +172,8 @@ export async function resolveInputFiles( workspaceId: string, inputFiles?: unknown[], inputTables?: unknown[], - inputDirectories?: unknown[] + inputDirectories?: unknown[], + chatId?: string ): Promise { const sandboxFiles: SandboxFile[] = [] const mounted: MountedBytes = { buffered: 0, url: 0 } @@ -182,9 +185,7 @@ export async function resolveInputFiles( `Too many input files (${inputFiles.length}). Maximum is ${MAX_MOUNTED_FILES}. Mount fewer files.` ) } - const allFiles = await listWorkspaceFiles(workspaceId, { - includeReservedSystemFiles: betaEnabled, - }) + let allFiles: WorkspaceFileRecord[] | undefined for (const fileRef of inputFiles) { const filePath = typeof fileRef === 'string' @@ -193,6 +194,35 @@ export async function resolveInputFiles( ? (fileRef as CanonicalFileInput).path : undefined if (!filePath) continue + const explicitSandboxPath = + typeof fileRef === 'object' && fileRef !== null + ? (fileRef as CanonicalFileInput).sandboxPath + : undefined + if (filePath.startsWith(UPLOADS_PREFIX)) { + const fileName = filePath.slice(UPLOADS_PREFIX.length) + if (!fileName || fileName.includes('/')) { + throw new Error(`Upload input path must identify one file: "${filePath}"`) + } + if (!chatId) { + throw new Error(`Chat context is required to mount upload: "${filePath}"`) + } + const record = await resolveChatUpload(fileName, chatId) + if (!record) { + throw new Error( + `Upload not found: "${filePath}". The chat or upload may no longer exist.` + ) + } + if (record.workspaceId !== workspaceId) { + throw new Error(`Upload does not belong to the current workspace: "${filePath}"`) + } + await pushWorkspaceFileMount( + sandboxFiles, + record, + explicitSandboxPath || `/home/user/${filePath}`, + mounted + ) + continue + } const alias = await resolveWorkflowAliasForWorkspace({ workspaceId, path: filePath }) if (!alias && isPlanAliasPath(filePath)) { logger.warn('Unsupported plan alias input file path', { filePath }) @@ -202,21 +232,15 @@ export async function resolveInputFiles( logger.warn('Input file is a plan alias directory', { filePath }) continue } + allFiles ??= await listWorkspaceFiles(workspaceId, { + includeReservedSystemFiles: betaEnabled, + }) const record = findWorkspaceFileRecord(allFiles, alias?.backingPath ?? filePath) if (!record) { - if (filePath.startsWith('uploads/')) { - throw new Error( - `Cannot mount "${filePath}": uploads/ files are not mountable into the sandbox. Use materialize_file to save it to a files/... path first, then mount that canonical path.` - ) - } throw new Error( `Input file not found: "${filePath}". Pass the exact canonical VFS path copied from glob/read (e.g. "files/Reports/data.csv").` ) } - const explicitSandboxPath = - typeof fileRef === 'object' && fileRef !== null - ? (fileRef as CanonicalFileInput).sandboxPath - : undefined const mountPath = explicitSandboxPath || (alias ? workflowAliasSandboxPath(alias.aliasPath) : getSandboxWorkspaceFilePath(record)) @@ -441,7 +465,8 @@ export async function executeFunctionExecute( context.workspaceId, inputFiles, inputTables, - inputDirectories + inputDirectories, + context.chatId ) if (resolved.length > 0) { const existing = (enrichedParams._sandboxFiles as SandboxFile[]) || [] From bb60a56768e47c1c1641aec02b68ec155ff7d9f5 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:18:26 -0700 Subject: [PATCH 04/10] chore(copilot): refresh Mothership tool contracts Regenerate the Sim catalog and runtime schemas from the finalized Mothership contract so canonical upload support and bounded media file declarations stay aligned across repositories. --- .../lib/copilot/generated/tool-catalog-v1.ts | 268 ++++++++++++++-- .../lib/copilot/generated/tool-schemas-v1.ts | 292 ++++++++++++++++-- 2 files changed, 500 insertions(+), 60 deletions(-) diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 20ed170b981..0deefda14be 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -1243,25 +1243,66 @@ export const Ffmpeg: ToolCatalogEntry = { height: { type: 'number', description: 'Target height in pixels (scale_pad).' }, inputs: { type: 'object', - description: 'Existing media files to use as generation or composition inputs.', + description: + 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', properties: { - files: { + directories: { type: 'array', description: - 'One or more media inputs from files/... or uploads/..., in operation order.', + 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', + }, + }, + required: ['path'], + }, + }, + files: { + type: 'array', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', }, }, required: ['path'], }, minItems: 1, }, + tables: { + type: 'array', + description: 'Workspace tables to mount as CSV files.', + items: { + type: 'object', + properties: { + path: { type: 'string', description: 'Canonical VFS table path when available.' }, + sandboxPath: { + type: 'string', + description: 'Optional full sandbox path for the mounted CSV.', + }, + tableId: { type: 'string', description: 'Workspace table ID.' }, + }, + }, + }, }, required: ['files'], }, @@ -1294,15 +1335,20 @@ export const Ffmpeg: ToolCatalogEntry = { outputs: { type: 'object', description: - 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', + 'Workspace files to create or overwrite from returned code results or sandbox-created files.', properties: { files: { type: 'array', description: - 'Exactly one file output. Parent folders must already exist for create mode.', + 'File outputs. Missing parent folders are created automatically for create mode.', items: { type: 'object', properties: { + format: { + type: 'string', + description: 'Optional serialization format for returned values.', + enum: ['json', 'csv', 'txt', 'md', 'html'], + }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1315,7 +1361,12 @@ export const Ffmpeg: ToolCatalogEntry = { path: { type: 'string', description: - 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', }, }, required: ['path', 'mode'], @@ -1405,14 +1456,14 @@ export const FunctionExecute: ToolCatalogEntry = { }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', }, sandboxPath: { type: 'string', @@ -1478,7 +1529,8 @@ export const FunctionExecute: ToolCatalogEntry = { }, path: { type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', + description: + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', }, sandboxPath: { type: 'string', @@ -1546,18 +1598,45 @@ export const GenerateAudio: ToolCatalogEntry = { }, inputs: { type: 'object', - description: 'Existing media files to use as generation or composition inputs.', + description: + 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', properties: { + directories: { + type: 'array', + description: + 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', + }, + }, + required: ['path'], + }, + }, files: { type: 'array', - description: 'Exactly one reference voice sample from files/... or uploads/....', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', }, }, required: ['path'], @@ -1565,6 +1644,21 @@ export const GenerateAudio: ToolCatalogEntry = { minItems: 1, maxItems: 1, }, + tables: { + type: 'array', + description: 'Workspace tables to mount as CSV files.', + items: { + type: 'object', + properties: { + path: { type: 'string', description: 'Canonical VFS table path when available.' }, + sandboxPath: { + type: 'string', + description: 'Optional full sandbox path for the mounted CSV.', + }, + tableId: { type: 'string', description: 'Workspace table ID.' }, + }, + }, + }, }, required: ['files'], }, @@ -1586,15 +1680,20 @@ export const GenerateAudio: ToolCatalogEntry = { outputs: { type: 'object', description: - 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', + 'Workspace files to create or overwrite from returned code results or sandbox-created files.', properties: { files: { type: 'array', description: - 'Exactly one file output. Parent folders must already exist for create mode.', + 'File outputs. Missing parent folders are created automatically for create mode.', items: { type: 'object', properties: { + format: { + type: 'string', + description: 'Optional serialization format for returned values.', + enum: ['json', 'csv', 'txt', 'md', 'html'], + }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1607,7 +1706,12 @@ export const GenerateAudio: ToolCatalogEntry = { path: { type: 'string', description: - 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', }, }, required: ['path', 'mode'], @@ -1651,39 +1755,86 @@ export const GenerateImage: ToolCatalogEntry = { }, inputs: { type: 'object', - description: 'Existing media files to use as generation or composition inputs.', + description: + 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', properties: { + directories: { + type: 'array', + description: + 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', + }, + }, + required: ['path'], + }, + }, files: { type: 'array', - description: 'One or more reference images to edit, combine, or restyle.', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', }, }, required: ['path'], }, minItems: 1, }, + tables: { + type: 'array', + description: 'Workspace tables to mount as CSV files.', + items: { + type: 'object', + properties: { + path: { type: 'string', description: 'Canonical VFS table path when available.' }, + sandboxPath: { + type: 'string', + description: 'Optional full sandbox path for the mounted CSV.', + }, + tableId: { type: 'string', description: 'Workspace table ID.' }, + }, + }, + }, }, required: ['files'], }, outputs: { type: 'object', description: - 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', + 'Workspace files to create or overwrite from returned code results or sandbox-created files.', properties: { files: { type: 'array', description: - 'Exactly one file output. Parent folders must already exist for create mode.', + 'File outputs. Missing parent folders are created automatically for create mode.', items: { type: 'object', properties: { + format: { + type: 'string', + description: 'Optional serialization format for returned values.', + enum: ['json', 'csv', 'txt', 'md', 'html'], + }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1696,7 +1847,12 @@ export const GenerateImage: ToolCatalogEntry = { path: { type: 'string', description: - 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', }, }, required: ['path', 'mode'], @@ -1743,18 +1899,45 @@ export const GenerateVideo: ToolCatalogEntry = { }, inputs: { type: 'object', - description: 'Existing media files to use as generation or composition inputs.', + description: + 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', properties: { + directories: { + type: 'array', + description: + 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', + }, + }, + required: ['path'], + }, + }, files: { type: 'array', - description: 'Exactly one start-frame image from files/... or uploads/....', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', }, }, required: ['path'], @@ -1762,6 +1945,21 @@ export const GenerateVideo: ToolCatalogEntry = { minItems: 1, maxItems: 1, }, + tables: { + type: 'array', + description: 'Workspace tables to mount as CSV files.', + items: { + type: 'object', + properties: { + path: { type: 'string', description: 'Canonical VFS table path when available.' }, + sandboxPath: { + type: 'string', + description: 'Optional full sandbox path for the mounted CSV.', + }, + tableId: { type: 'string', description: 'Workspace table ID.' }, + }, + }, + }, }, required: ['files'], }, @@ -1789,15 +1987,20 @@ export const GenerateVideo: ToolCatalogEntry = { outputs: { type: 'object', description: - 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', + 'Workspace files to create or overwrite from returned code results or sandbox-created files.', properties: { files: { type: 'array', description: - 'Exactly one file output. Parent folders must already exist for create mode.', + 'File outputs. Missing parent folders are created automatically for create mode.', items: { type: 'object', properties: { + format: { + type: 'string', + description: 'Optional serialization format for returned values.', + enum: ['json', 'csv', 'txt', 'md', 'html'], + }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1810,7 +2013,12 @@ export const GenerateVideo: ToolCatalogEntry = { path: { type: 'string', description: - 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', }, }, required: ['path', 'mode'], @@ -3243,14 +3451,14 @@ export const RunCode: ToolCatalogEntry = { }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', }, sandboxPath: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index 15d51724314..ca42d6561b8 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -1058,25 +1058,72 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: 'Existing media files to use as generation or composition inputs.', + description: + 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', properties: { - files: { + directories: { type: 'array', description: - 'One or more media inputs from files/... or uploads/..., in operation order.', + 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', + 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', + }, + }, + required: ['path'], + }, + }, + files: { + type: 'array', + description: 'Workspace files or chat uploads to mount into the sandbox.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', }, }, required: ['path'], }, minItems: 1, }, + tables: { + type: 'array', + description: 'Workspace tables to mount as CSV files.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Canonical VFS table path when available.', + }, + sandboxPath: { + type: 'string', + description: 'Optional full sandbox path for the mounted CSV.', + }, + tableId: { + type: 'string', + description: 'Workspace table ID.', + }, + }, + }, + }, }, required: ['files'], }, @@ -1109,15 +1156,20 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', + 'Workspace files to create or overwrite from returned code results or sandbox-created files.', properties: { files: { type: 'array', description: - 'Exactly one file output. Parent folders must already exist for create mode.', + 'File outputs. Missing parent folders are created automatically for create mode.', items: { type: 'object', properties: { + format: { + type: 'string', + description: 'Optional serialization format for returned values.', + enum: ['json', 'csv', 'txt', 'md', 'html'], + }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1130,7 +1182,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { path: { type: 'string', description: - 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', }, }, required: ['path', 'mode'], @@ -1217,14 +1274,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', }, sandboxPath: { type: 'string', @@ -1296,7 +1353,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, path: { type: 'string', - description: 'Canonical destination VFS path, e.g. "files/Reports/chart.png".', + description: + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', }, sandboxPath: { type: 'string', @@ -1354,18 +1412,45 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: 'Existing media files to use as generation or composition inputs.', + description: + 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', properties: { + directories: { + type: 'array', + description: + 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', + }, + }, + required: ['path'], + }, + }, files: { type: 'array', - description: 'Exactly one reference voice sample from files/... or uploads/....', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', }, }, required: ['path'], @@ -1373,6 +1458,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { minItems: 1, maxItems: 1, }, + tables: { + type: 'array', + description: 'Workspace tables to mount as CSV files.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Canonical VFS table path when available.', + }, + sandboxPath: { + type: 'string', + description: 'Optional full sandbox path for the mounted CSV.', + }, + tableId: { + type: 'string', + description: 'Workspace table ID.', + }, + }, + }, + }, }, required: ['files'], }, @@ -1394,15 +1500,20 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', + 'Workspace files to create or overwrite from returned code results or sandbox-created files.', properties: { files: { type: 'array', description: - 'Exactly one file output. Parent folders must already exist for create mode.', + 'File outputs. Missing parent folders are created automatically for create mode.', items: { type: 'object', properties: { + format: { + type: 'string', + description: 'Optional serialization format for returned values.', + enum: ['json', 'csv', 'txt', 'md', 'html'], + }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1415,7 +1526,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { path: { type: 'string', description: - 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', }, }, required: ['path', 'mode'], @@ -1456,39 +1572,92 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: 'Existing media files to use as generation or composition inputs.', + description: + 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', properties: { + directories: { + type: 'array', + description: + 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', + }, + }, + required: ['path'], + }, + }, files: { type: 'array', - description: 'One or more reference images to edit, combine, or restyle.', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', }, }, required: ['path'], }, minItems: 1, }, + tables: { + type: 'array', + description: 'Workspace tables to mount as CSV files.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Canonical VFS table path when available.', + }, + sandboxPath: { + type: 'string', + description: 'Optional full sandbox path for the mounted CSV.', + }, + tableId: { + type: 'string', + description: 'Workspace table ID.', + }, + }, + }, + }, }, required: ['files'], }, outputs: { type: 'object', description: - 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', + 'Workspace files to create or overwrite from returned code results or sandbox-created files.', properties: { files: { type: 'array', description: - 'Exactly one file output. Parent folders must already exist for create mode.', + 'File outputs. Missing parent folders are created automatically for create mode.', items: { type: 'object', properties: { + format: { + type: 'string', + description: 'Optional serialization format for returned values.', + enum: ['json', 'csv', 'txt', 'md', 'html'], + }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1501,7 +1670,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { path: { type: 'string', description: - 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', }, }, required: ['path', 'mode'], @@ -1542,18 +1716,45 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, inputs: { type: 'object', - description: 'Existing media files to use as generation or composition inputs.', + description: + 'Workspace resources to mount into the sandbox. Copy paths verbatim from glob/read/grep output — they are percent-encoded per segment (spaces are %20, an in-name slash is %2F; parentheses and dots stay literal). Both the encoded path and the plain name resolve, so copy the returned path exactly rather than retyping or decoding it.', properties: { + directories: { + type: 'array', + description: + 'Workspace folders to mount recursively into the sandbox, including nested files and empty folders.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: + 'Canonical VFS folder path, e.g. "files/Reports". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full sandbox directory path override. Omit to mount at /home/user/{path}.', + }, + }, + required: ['path'], + }, + }, files: { type: 'array', - description: 'Exactly one start-frame image from files/... or uploads/....', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Input VFS path copied verbatim from glob/read, using either persistent "files/..." or chat-scoped "uploads/...".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', + }, + sandboxPath: { + type: 'string', + description: + 'Full sandbox path to mount at, e.g. /home/user/inputs/data.csv. STRONGLY RECOMMENDED whenever the file name has spaces or special characters: the default mount path is the percent-ENCODED canonical path (e.g. /home/user/files/Q4%20Sales%20(Final).csv), which code using the human-readable name will not find. Set a simple sandboxPath and read exactly that.', }, }, required: ['path'], @@ -1561,6 +1762,27 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { minItems: 1, maxItems: 1, }, + tables: { + type: 'array', + description: 'Workspace tables to mount as CSV files.', + items: { + type: 'object', + properties: { + path: { + type: 'string', + description: 'Canonical VFS table path when available.', + }, + sandboxPath: { + type: 'string', + description: 'Optional full sandbox path for the mounted CSV.', + }, + tableId: { + type: 'string', + description: 'Workspace table ID.', + }, + }, + }, + }, }, required: ['files'], }, @@ -1588,15 +1810,20 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { outputs: { type: 'object', description: - 'One persistent workspace file to create or overwrite. Chat uploads are read-only inputs and cannot be output targets.', + 'Workspace files to create or overwrite from returned code results or sandbox-created files.', properties: { files: { type: 'array', description: - 'Exactly one file output. Parent folders must already exist for create mode.', + 'File outputs. Missing parent folders are created automatically for create mode.', items: { type: 'object', properties: { + format: { + type: 'string', + description: 'Optional serialization format for returned values.', + enum: ['json', 'csv', 'txt', 'md', 'html'], + }, mimeType: { type: 'string', description: 'Optional MIME type override when inference is not enough.', @@ -1609,7 +1836,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { path: { type: 'string', description: - 'Persistent destination VFS path starting with "files/", e.g. "files/Reports/chart.png". "uploads/" paths are invalid output targets.', + 'Canonical destination VFS path under "files/", e.g. "files/Reports/chart.png".', + }, + sandboxPath: { + type: 'string', + description: + 'Optional full path to a file created inside the sandbox. Omit to save the code return value.', }, }, required: ['path', 'mode'], @@ -3022,14 +3254,14 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, files: { type: 'array', - description: 'Workspace files to mount into the sandbox.', + description: 'Workspace files or chat uploads to mount into the sandbox.', items: { type: 'object', properties: { path: { type: 'string', description: - 'Canonical VFS file path, e.g. "files/Reports/sales.csv". By default this mounts at "/home/user/{path}".', + 'Canonical VFS file path, e.g. "files/Reports/sales.csv" or "uploads/sales.csv". By default this mounts at "/home/user/{path}".', }, sandboxPath: { type: 'string', From 3b10b7391601e95320c44147410865f4d46f6e81 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:54:46 -0700 Subject: [PATCH 05/10] refactor(media): clarify file cardinality helpers --- apps/sim/lib/copilot/tools/server/image/generate-image.ts | 4 ++-- apps/sim/lib/copilot/tools/server/media/file-paths.test.ts | 6 +++--- apps/sim/lib/copilot/tools/server/media/file-paths.ts | 6 +++--- apps/sim/lib/copilot/tools/server/media/generate-audio.ts | 4 ++-- apps/sim/lib/copilot/tools/server/media/generate-video.ts | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.ts index 0cda9700a22..dcea0fd7cda 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.ts @@ -9,7 +9,7 @@ import { } from '@/lib/copilot/tools/server/base-tool' import { prepareMediaOutput, - requireMediaFileDeclarations, + requireAtLeastOneMediaFile, resolveMediaInputFile, } from '@/lib/copilot/tools/server/media/file-paths' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' @@ -91,7 +91,7 @@ export const generateImageServerTool: BaseServerTool file.path) + ? requireAtLeastOneMediaFile(params.inputs.files, 'Input').map((file) => file.path) : [] if (referencePaths.length) { diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts index f757e960f0a..6bb0b5284ce 100644 --- a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts @@ -22,8 +22,8 @@ vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ })) import { - getSingleMediaFileDeclaration, prepareMediaOutput, + requireExactlyOneMediaFile, resolveMediaInputFile, validateMediaOutputFile, } from '@/lib/copilot/tools/server/media/file-paths' @@ -164,9 +164,9 @@ describe('media file paths', () => { it('rejects extra declarations for single-file media parameters', () => { expect(() => - getSingleMediaFileDeclaration([{ path: 'files/a.png' }, { path: 'files/b.png' }], 'Output') + requireExactlyOneMediaFile([{ path: 'files/a.png' }, { path: 'files/b.png' }], 'Output') ).toThrow('Output requires exactly one file; received 2') - expect(() => getSingleMediaFileDeclaration([], 'Output')).toThrow( + expect(() => requireExactlyOneMediaFile([], 'Output')).toThrow( 'Output requires exactly one file; received 0' ) }) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.ts index ec07aab70e6..74d684af8c9 100644 --- a/apps/sim/lib/copilot/tools/server/media/file-paths.ts +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.ts @@ -17,7 +17,7 @@ interface MediaOutputDeclaration { mimeType?: string } -export function getSingleMediaFileDeclaration( +export function requireExactlyOneMediaFile( files: T[] | undefined, label: 'Input' | 'Output' ): T { @@ -27,7 +27,7 @@ export function getSingleMediaFileDeclaration( return files[0] } -export function requireMediaFileDeclarations( +export function requireAtLeastOneMediaFile( files: T[] | undefined, label: 'Input' | 'Output' ): T[] { @@ -94,7 +94,7 @@ export async function prepareMediaOutput(args: workspaceId: string userId: string }): Promise { - const file = getSingleMediaFileDeclaration(args.output?.files, 'Output') + const file = requireExactlyOneMediaFile(args.output?.files, 'Output') const output = { ...file, mode: file.mode ?? 'create' } await validateMediaOutputFile({ workspaceId: args.workspaceId, diff --git a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts index 5b000d67c16..f3ee7cd88d6 100644 --- a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts +++ b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts @@ -7,8 +7,8 @@ import { type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' import { - getSingleMediaFileDeclaration, prepareMediaOutput, + requireExactlyOneMediaFile, resolveMediaInputFile, } from '@/lib/copilot/tools/server/media/file-paths' import { writeWorkspaceFileByPath } from '@/lib/copilot/vfs/resource-writer' @@ -82,7 +82,7 @@ export const generateAudioServerTool: BaseServerTool Date: Wed, 22 Jul 2026 18:10:14 -0700 Subject: [PATCH 06/10] fix(media): restore automatic ffmpeg output paths --- .../lib/copilot/tools/server/media/ffmpeg.ts | 13 ++++------- .../media/media-tools-validation.test.ts | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/copilot/tools/server/media/ffmpeg.ts b/apps/sim/lib/copilot/tools/server/media/ffmpeg.ts index bba941fcff5..eedebe7a38b 100644 --- a/apps/sim/lib/copilot/tools/server/media/ffmpeg.ts +++ b/apps/sim/lib/copilot/tools/server/media/ffmpeg.ts @@ -83,7 +83,7 @@ export const ffmpegServerTool: BaseServerTool = { try { const outputFile = - params.operation === 'probe' + params.operation === 'probe' || !params.outputs?.files?.length ? undefined : await prepareMediaOutput({ output: params.outputs, @@ -130,21 +130,18 @@ export const ffmpegServerTool: BaseServerTool = { } } - if (!result.buffer) { + if (!result.buffer || !result.ext) { return { success: false, message: `ffmpeg ${params.operation} produced no output` } } - if (!outputFile) { - return { success: false, message: `ffmpeg ${params.operation} requires an output file` } - } - const outputPath = outputFile.path - const mode = outputFile.mode + const outputPath = outputFile?.path ?? `files/ffmpeg-${params.operation}.${result.ext}` + const mode = outputFile?.mode ?? 'create' assertServerToolNotAborted(context) const written = await writeWorkspaceFileByPath({ workspaceId, userId: context.userId, - target: { path: outputPath, mode, mimeType: outputFile.mimeType }, + target: { path: outputPath, mode, mimeType: outputFile?.mimeType }, buffer: result.buffer, inferredMimeType: result.contentType || 'application/octet-stream', }) diff --git a/apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts b/apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts index ef6c1055d9c..a24bdccaad7 100644 --- a/apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts +++ b/apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts @@ -193,6 +193,29 @@ describe('media tool path validation', () => { ) }) + it('uses the default ffmpeg output path when outputs are omitted', async () => { + mocks.resolveWorkspaceFileReference.mockResolvedValue({ + ...UPLOAD_RECORD, + name: 'input.mov', + type: 'video/quicktime', + }) + + const result = await ffmpegServerTool.execute( + { + operation: 'convert', + inputs: { files: [{ path: 'files/input.mov' }] }, + }, + CONTEXT + ) + + expect(result.success).toBe(true) + expect(mocks.writeWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: { path: 'files/ffmpeg-convert.mp4', mode: 'create', mimeType: undefined }, + }) + ) + }) + it('rejects extra video inputs before calling the provider', async () => { const result = await generateVideoServerTool.execute( { From 522855a67088c0364ae5467c5733a8e74a0555b7 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:29:01 -0700 Subject: [PATCH 07/10] fix(media): restore automatic generated output paths --- .../lib/copilot/generated/tool-catalog-v1.ts | 6 ++-- .../lib/copilot/generated/tool-schemas-v1.ts | 6 ++-- .../tools/server/image/generate-image.test.ts | 22 ++++++++------ .../tools/server/image/generate-image.ts | 21 ++++++++------ .../tools/server/media/generate-audio.ts | 29 ++++++++++++++----- .../tools/server/media/generate-video.ts | 18 +++++++----- .../media/media-tools-validation.test.ts | 21 ++++++++++++-- 7 files changed, 80 insertions(+), 43 deletions(-) diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 0deefda14be..221083c728e 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -1734,7 +1734,7 @@ export const GenerateAudio: ToolCatalogEntry = { }, voice: { type: 'string', description: 'Optional voice name or id for speech.' }, }, - required: ['prompt', 'outputs'], + required: ['prompt'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], @@ -1869,7 +1869,7 @@ export const GenerateImage: ToolCatalogEntry = { 'Detailed text description of the image to generate, or editing instructions when editing the image(s) passed in `inputs.files`.', }, }, - required: ['prompt', 'outputs'], + required: ['prompt'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], @@ -2044,7 +2044,7 @@ export const GenerateVideo: ToolCatalogEntry = { enum: ['720p', '1080p', '4k'], }, }, - required: ['prompt', 'outputs'], + required: ['prompt'], }, requiredPermission: 'write', capabilities: ['file_input', 'file_output', 'generated_media'], diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index ca42d6561b8..e586abe380d 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -1557,7 +1557,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { description: 'Optional voice name or id for speech.', }, }, - required: ['prompt', 'outputs'], + required: ['prompt'], }, resultSchema: undefined, }, @@ -1692,7 +1692,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'Detailed text description of the image to generate, or editing instructions when editing the image(s) passed in `inputs.files`.', }, }, - required: ['prompt', 'outputs'], + required: ['prompt'], }, resultSchema: undefined, }, @@ -1867,7 +1867,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { enum: ['720p', '1080p', '4k'], }, }, - required: ['prompt', 'outputs'], + required: ['prompt'], }, resultSchema: undefined, }, diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts index d5e7e5aed8a..e97cc4f52f2 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.test.ts @@ -170,7 +170,7 @@ describe('generateImageServerTool', () => { expect(mocks.generateContent).not.toHaveBeenCalled() }) - it('rejects explicitly empty input and output containers before generation', async () => { + it('rejects an explicitly empty input container before generation', async () => { const emptyInputs = await generateImageServerTool.execute( { prompt: 'Edit the supplied image', @@ -179,22 +179,26 @@ describe('generateImageServerTool', () => { }, CONTEXT ) - const emptyOutputs = await generateImageServerTool.execute( - { prompt: 'Create a portrait', outputs: {} }, - CONTEXT - ) expect(emptyInputs).toEqual({ success: false, message: 'Failed to generate image: Input requires at least one file', }) - expect(emptyOutputs).toEqual({ - success: false, - message: 'Failed to generate image: Output requires exactly one file; received 0', - }) expect(mocks.generateContent).not.toHaveBeenCalled() }) + it('uses the default image output path when outputs are omitted', async () => { + const result = await generateImageServerTool.execute({ prompt: 'Create a portrait' }, CONTEXT) + + expect(result.success).toBe(true) + expect(mocks.validateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mocks.writeWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: { path: 'files/generated-image.png', mode: 'create', mimeType: undefined }, + }) + ) + }) + it('fails before generation when canonical output preflight rejects a files/ target', async () => { mocks.validateWorkspaceFileWriteTarget.mockRejectedValue( new Error('File already exists at files/result.png') diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.ts index dcea0fd7cda..989dd609d62 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.ts @@ -76,11 +76,13 @@ export const generateImageServerTool: BaseServerTool = { name: GenerateAudio.id, @@ -73,11 +83,13 @@ export const generateAudioServerTool: BaseServerTool { expect(mocks.writeWorkspaceFileByPath).not.toHaveBeenCalled() }) - it('rejects missing generated-media outputs before calling the provider', async () => { + it('uses the default video output path when outputs are omitted', async () => { const result = await generateVideoServerTool.execute( { prompt: 'Create a launch video' }, CONTEXT ) - expect(result.success).toBe(false) - expect(mocks.generateFalVideo).not.toHaveBeenCalled() + expect(result.success).toBe(true) + expect(mocks.writeWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: { path: 'files/generated-video.mp4', mode: 'create', mimeType: undefined }, + }) + ) + }) + + it('uses a content-type-derived default audio output path when outputs are omitted', async () => { + const result = await generateAudioServerTool.execute({ prompt: 'Create a voiceover' }, CONTEXT) + + expect(result.success).toBe(true) + expect(mocks.writeWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: { path: 'files/generated-audio.mp3', mode: 'create', mimeType: undefined }, + }) + ) }) it('passes uploads/ media files to ffmpeg', async () => { From bcf65d0c4b7f3efd6dcd9b341dff314a34f5c006 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:52:18 -0700 Subject: [PATCH 08/10] refactor(media): clarify validation ordering --- apps/sim/lib/copilot/tools/handlers/function-execute.ts | 1 + apps/sim/lib/copilot/tools/server/image/generate-image.ts | 1 + apps/sim/lib/copilot/tools/server/media/ffmpeg.ts | 1 + apps/sim/lib/copilot/tools/server/media/generate-audio.ts | 3 ++- apps/sim/lib/copilot/tools/server/media/generate-video.ts | 3 ++- 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index 010a2d24791..ee5d1b21fd6 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -232,6 +232,7 @@ export async function resolveInputFiles( logger.warn('Input file is a plan alias directory', { filePath }) continue } + // Upload mounts bypass the workspace listing, so load it only for persistent files. allFiles ??= await listWorkspaceFiles(workspaceId, { includeReservedSystemFiles: betaEnabled, }) diff --git a/apps/sim/lib/copilot/tools/server/image/generate-image.ts b/apps/sim/lib/copilot/tools/server/image/generate-image.ts index 989dd609d62..ca0f04dd215 100644 --- a/apps/sim/lib/copilot/tools/server/image/generate-image.ts +++ b/apps/sim/lib/copilot/tools/server/image/generate-image.ts @@ -76,6 +76,7 @@ export const generateImageServerTool: BaseServerTool = { } try { + // Preflight explicit outputs so invalid paths fail before media processing begins. const outputFile = params.operation === 'probe' || !params.outputs?.files?.length ? undefined diff --git a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts index 2bb6b923aa2..e2f6c1abdd5 100644 --- a/apps/sim/lib/copilot/tools/server/media/generate-audio.ts +++ b/apps/sim/lib/copilot/tools/server/media/generate-audio.ts @@ -83,6 +83,7 @@ export const generateAudioServerTool: BaseServerTool Date: Wed, 22 Jul 2026 19:02:01 -0700 Subject: [PATCH 09/10] refactor(media): restore ffmpeg fallback expression --- apps/sim/lib/copilot/tools/server/media/ffmpeg.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/copilot/tools/server/media/ffmpeg.ts b/apps/sim/lib/copilot/tools/server/media/ffmpeg.ts index 678b910b0a4..2af948cd741 100644 --- a/apps/sim/lib/copilot/tools/server/media/ffmpeg.ts +++ b/apps/sim/lib/copilot/tools/server/media/ffmpeg.ts @@ -135,7 +135,7 @@ export const ffmpegServerTool: BaseServerTool = { return { success: false, message: `ffmpeg ${params.operation} produced no output` } } - const outputPath = outputFile?.path ?? `files/ffmpeg-${params.operation}.${result.ext}` + const outputPath = outputFile?.path || `files/ffmpeg-${params.operation}.${result.ext}` const mode = outputFile?.mode ?? 'create' assertServerToolNotAborted(context) From 3e1bdbc423a5059b8e39e1a339b866da4c5266a6 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Wed, 22 Jul 2026 19:13:39 -0700 Subject: [PATCH 10/10] Address PR review feedback (#5615) - Reject chat uploads that belong to another workspace - Add regression coverage for media upload ownership --- .../tools/server/media/file-paths.test.ts | 16 +++++++++++++++- .../lib/copilot/tools/server/media/file-paths.ts | 3 +++ .../server/media/media-tools-validation.test.ts | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts index 6bb0b5284ce..4193c038fbe 100644 --- a/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.test.ts @@ -28,7 +28,7 @@ import { validateMediaOutputFile, } from '@/lib/copilot/tools/server/media/file-paths' -const FILE_RECORD = { id: 'wf_file', name: 'portrait.png' } +const FILE_RECORD = { id: 'wf_file', name: 'portrait.png', workspaceId: 'workspace-1' } describe('media file paths', () => { beforeEach(() => { @@ -65,6 +65,20 @@ describe('media file paths', () => { expect(mocks.resolveChatUpload).toHaveBeenCalledWith('My%20Portrait.png', 'chat-1') }) + it('rejects uploads from a different workspace', async () => { + mocks.resolveChatUpload.mockResolvedValue({ ...FILE_RECORD, workspaceId: 'workspace-2' }) + + await expect( + resolveMediaInputFile({ + workspaceId: 'workspace-1', + chatId: 'chat-1', + path: 'uploads/My%20Portrait.png', + }) + ).rejects.toThrow( + 'Upload does not belong to the current workspace: "uploads/My%20Portrait.png"' + ) + }) + it('rejects unresolved inputs instead of dropping them', async () => { mocks.resolveWorkspaceFileReference.mockResolvedValue(null) diff --git a/apps/sim/lib/copilot/tools/server/media/file-paths.ts b/apps/sim/lib/copilot/tools/server/media/file-paths.ts index 74d684af8c9..eca35fdab73 100644 --- a/apps/sim/lib/copilot/tools/server/media/file-paths.ts +++ b/apps/sim/lib/copilot/tools/server/media/file-paths.ts @@ -57,6 +57,9 @@ export async function resolveMediaInputFile(args: { throw new Error(`Chat context is required for upload input: ${path}`) } file = await resolveChatUpload(filename, args.chatId) + if (file && file.workspaceId !== args.workspaceId) { + throw new Error(`Upload does not belong to the current workspace: "${path}"`) + } } else { file = await resolveWorkspaceFileReference(args.workspaceId, path) } diff --git a/apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts b/apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts index c78facb76aa..450300d5ba7 100644 --- a/apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts +++ b/apps/sim/lib/copilot/tools/server/media/media-tools-validation.test.ts @@ -38,6 +38,7 @@ const UPLOAD_RECORD = { id: 'wf_upload', name: 'start.png', type: 'image/png', + workspaceId: 'workspace-1', storageContext: 'mothership', }