yiliang114 · Sep 23, 2026 · Sep 2, 2026 · Sep 3, 2026 · Sep 3, 2026 · Sep 3, 2026
@@ -5,6 +5,7 @@
  */
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
+import sharp from 'sharp';
 import type { Mocked } from 'vitest';
 import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
 import { safeJsonStringify } from '../utils/safeJsonStringify.js';
@@ -724,6 +725,107 @@ describe('DiscoveredMCPTool', () => {
       );
     });
 
+    it('bounds an oversized image returned by an MCP tool', async () => {
+      const oversized = await sharp({
+        create: {
+          width: 3840,
+          height: 2160,
+          channels: 3,
+          background: '#204080',
+        },
+      })
+        .png()
+        .toBuffer();
+      mockCallTool.mockResolvedValue([
+        {
+          functionResponse: {
+            name: serverToolName,
+            response: {
+              content: [
+                {
+                  type: 'image',
+                  data: oversized.toString('base64'),
+                  mimeType: 'image/png',
+                },
+              ],
+            },
+          },
+        },
+      ] as Part[]);
+
+      const invocation = tool.build({ param: 'screenshot' });
+      const toolResult = await invocation.execute(new AbortController().signal);
+
+      const parts = toolResult.llmContent as Part[];
+      const inline = parts[1]!.inlineData!;
+      expect(inline.mimeType).toBe('image/jpeg');
+      const bounded = await sharp(
+        Buffer.from(inline.data!, 'base64'),
+      ).metadata();
+      expect(Math.max(bounded.width, bounded.height)).toBeLessThanOrEqual(1568);
+      expect(
+        Math.ceil(bounded.width / 28) * Math.ceil(bounded.height / 28),
+      ).toBeLessThanOrEqual(1568);
+    });
+
+    it('leaves an in-budget image from an MCP tool untouched', async () => {
+      const small = await sharp({
+        create: { width: 200, height: 100, channels: 4, background: '#204080' },
+      })
+        .png()
+        .toBuffer();
+      const data = small.toString('base64');
+      mockCallTool.mockResolvedValue([
+        {
+          functionResponse: {
+            name: serverToolName,
+            response: {
+              content: [{ type: 'image', data, mimeType: 'image/png' }],
+            },
+          },
+        },
+      ] as Part[]);
+
+      const invocation = tool.build({ param: 'icon' });
+      const toolResult = await invocation.execute(new AbortController().signal);
+
+      const parts = toolResult.llmContent as Part[];
+      expect(parts[1]!.inlineData).toEqual({ mimeType: 'image/png', data });
+    });
+
+    it('forwards an image the renderer cannot bound unchanged', async () => {
+      const animated = await sharp({
+        create: {
+          width: 3840,
+          height: 2160,
+          channels: 3,
+          background: '#204080',
+        },
+      })
+        .gif()
+        .toBuffer();
+      const data = animated.toString('base64');
+      mockCallTool.mockResolvedValue([
+        {
+          functionResponse: {
+            name: serverToolName,
+            response: {
+              content: [{ type: 'image', data, mimeType: 'image/gif' }],
+            },
+          },
+        },
+      ] as Part[]);
+
+      const invocation = tool.build({ param: 'gif' });
+      const toolResult = await invocation.execute(new AbortController().signal);
+
+      const parts = toolResult.llmContent as Part[];
+      expect(parts[1]!.inlineData).toEqual({
+        mimeType: 'image/gif',
+        data,
+      });
+    });
+
     it('should ignore unknown content block types', async () => {
       const params = { param: 'test' };
       const sdkResponse: Part[] = [

@@ -31,6 +31,7 @@ import { StructuredToolError, ToolErrorType } from './tool-error.js';
 import type { Config } from '../config/config.js';
 import { truncateToolOutput } from './truncation.js';
 import { createDebugLogger } from '../utils/debugLogger.js';
+import { boundImageBuffer, ImageViewError } from '../utils/image-view.js';
 import { getErrorMessage, isAbortError } from '../utils/errors.js';
 import {
   getAllMCPServerStatuses,
@@ -697,13 +698,20 @@ class DiscoveredMCPToolInvocation extends BaseToolInvocation<
       );
 
       if (this.isMCPToolError(rawResponseParts)) {
-        return await this.buildMcpToolError(rawResponseParts, {
-          name: this.serverToolName,
-          args: this.params,
-        });
+        return await this.buildMcpToolError(
+          rawResponseParts,
+          {
+            name: this.serverToolName,
+            args: this.params,
+          },
+          signal,
+        );
       }
 
-      const transformedParts = transformMcpContentToParts(rawResponseParts);
+      const transformedParts = await boundInlineImageParts(
+        transformMcpContentToParts(rawResponseParts),
+        signal,
+      );
       const truncated = await this.truncateTextParts(transformedParts);
       const fallbackText = getDisplayFromPartsWithPersistedOutput(
         transformedParts,
@@ -863,10 +871,17 @@ class DiscoveredMCPToolInvocation extends BaseToolInvocation<
       const rawResponseParts = outcome;
 
       if (this.isMCPToolError(rawResponseParts)) {
-        return await this.buildMcpToolError(rawResponseParts, functionCalls[0]);
+        return await this.buildMcpToolError(
+          rawResponseParts,
+          functionCalls[0],
+          signal,
+        );
       }
 
-      const transformedParts = transformMcpContentToParts(rawResponseParts);
+      const transformedParts = await boundInlineImageParts(
+        transformMcpContentToParts(rawResponseParts),
+        signal,
+      );
       const truncated = await this.truncateTextParts(transformedParts);
 
       return {
@@ -893,13 +908,16 @@ class DiscoveredMCPToolInvocation extends BaseToolInvocation<
   private async buildMcpToolError(
     rawResponseParts: Part[],
     functionCall: FunctionCall,
+    signal: AbortSignal,
   ): Promise<ToolResult> {
     const imageContent = getMcpErrorImageContent(rawResponseParts);
     let llmContent: PartListUnion;
     let errorMessage: string;
     let persistedOutputFiles: string[] | undefined;
     if (imageContent) {
-      const truncatedContent = await this.truncateTextParts(imageContent);
+      const truncatedContent = await this.truncateTextParts(
+        await boundInlineImageParts(imageContent, signal),
+      );
       llmContent = truncatedContent.parts;
       persistedOutputFiles = truncatedContent.persistedOutputFiles;
       errorMessage = `MCP tool '${
@@ -1284,6 +1302,51 @@ function transformImageAudioBlock(
   ];
 }
 
+/**
+ * Shrink oversized inline images to the same visual budget `read_file`
+ * applies, so a full-resolution screenshot from a browser automation server
+ * does not enter the conversation verbatim. Images that already fit, and any
+ * the renderer cannot handle, are forwarded unchanged.
+ */
+async function boundInlineImageParts(
+  parts: Part[],
+  signal: AbortSignal,
+): Promise<Part[]> {
+  return Promise.all(
+    parts.map(async (part) => {
+      const inline = part.inlineData;
+      if (!inline?.data || !inline.mimeType?.startsWith('image/')) {
+        return part;
+      }
+      try {
+        const view = await boundImageBuffer(
+          Buffer.from(inline.data, 'base64'),
+          inline.mimeType,
+          signal,
+        );
+        if (!view) {
+          return part;
+        }
+        return {
+          inlineData: {
+            ...inline,
+            data: view.bytes.toString('base64'),
+            mimeType: view.mimeType,
+          },
+        };
+      } catch (error) {
+        if (error instanceof ImageViewError) {
+          debugLogger.debug(
+            `Forwarding MCP image unbounded: ${getErrorMessage(error)}`,
+          );
+          return part;
+        }
+        throw error;
+      }
+    }),
+  );
+}
+
 function transformResourceBlock(
   block: McpResourceBlock,
   toolName: string,

@@ -10,6 +10,7 @@ import path from 'node:path';
 import sharp from 'sharp';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 import {
+  boundImageBuffer,
   orientedSize,
   renderImageOverview,
   renderNormalizedImageCrop,
@@ -112,6 +113,50 @@ describe('image views', () => {
     expect(view.bytes.length).toBeLessThanOrEqual(9 * 1024 * 1024);
   });
 
+  it('leaves an in-budget image buffer untouched', async () => {
+    const bytes = await sharp({
+      create: { width: 200, height: 100, channels: 3, background: '#306090' },
+    })
+      .png()
+      .toBuffer();
+
+    await expect(boundImageBuffer(bytes, 'image/png', signal)).resolves.toBe(
+      null,
+    );
+  });
+
+  it('bounds an oversized image buffer to the shared budget', async () => {
+    const bytes = await sharp({
+      create: { width: 3840, height: 2160, channels: 3, background: '#804020' },
+    })
+      .png()
+      .toBuffer();
+
+    const view = await boundImageBuffer(bytes, 'image/png', signal);
+
+    expect(view).not.toBe(null);
+    expect(view!.mimeType).toBe('image/jpeg');
+    expect(Math.max(view!.outputWidth, view!.outputHeight)).toBeLessThanOrEqual(
+      1568,
+    );
+    expect(
+      Math.ceil(view!.outputWidth / 28) * Math.ceil(view!.outputHeight / 28),
+    ).toBeLessThanOrEqual(1568);
+    expect(view!.bytes.length).toBeLessThan(bytes.length);
+  });
+
+  it('reports unsupported_image for a format the renderer cannot bound', async () => {
+    const bytes = await sharp({
+      create: { width: 3840, height: 2160, channels: 3, background: '#804020' },
+    })
+      .gif()
+      .toBuffer();
+
+    await expect(
+      boundImageBuffer(bytes, 'image/gif', signal),
+    ).rejects.toMatchObject({ code: 'unsupported_image' });
+  });
+
   it('reports decode_failed for a corrupt canonical image', async () => {
     const filePath = path.join(root, 'corrupt.png');
     await fs.writeFile(filePath, 'not a real png');