Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 12 additions & 76 deletions packages/app/src/composer/attachments/attachments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,64 +4,6 @@ import { createBlobReference } from "@/runtime/persistence/drafts"
import { uuid } from "@/runtime/persistence/uuid"
import type { ComposerAttachment, ComposerPrompt } from "../types"

const accepted = [
"image/png",
"image/jpeg",
"image/gif",
"image/webp",
"application/pdf",
"text/*",
"application/json",
"application/ld+json",
"application/toml",
"application/x-toml",
"application/x-yaml",
"application/xml",
"application/yaml",
".c",
".cc",
".cjs",
".conf",
".cpp",
".css",
".csv",
".cts",
".env",
".go",
".gql",
".graphql",
".h",
".hh",
".hpp",
".htm",
".html",
".ini",
".java",
".js",
".json",
".jsx",
".log",
".md",
".mdx",
".mjs",
".mts",
".py",
".rb",
".rs",
".sass",
".scss",
".sh",
".sql",
".toml",
".ts",
".tsx",
".txt",
".xml",
".yaml",
".yml",
".zsh",
]

type PromptTarget = {
current: () => ComposerPrompt
cursor: () => number | undefined
Expand All @@ -75,7 +17,6 @@ export type ComposerAttachmentConfig = {
) => Promise<void>
directory: () => string
isDialogActive: () => boolean
warn: () => void
duplicate: () => void
onError: (error: unknown) => void
readClipboardImage?: () => Promise<File | null>
Expand All @@ -102,13 +43,9 @@ export function createComposerAttachments(
if (!editor) return
return { prompt, cursor: prompt.cursor() ?? cursorPosition(editor) }
}
const add = async (file: File, toast = true, target = capture(), clipboard = false) => {
const add = async (file: File, target = capture(), clipboard = false) => {
if (!target) return false
const mime = await attachmentMime(file)
if (!mime) {
if (toast) input.warn()
return false
}
const blob = input.store ? await input.store(file) : await createBlobReference(file)
const sourcePath = input.getPathForFile?.(file) || undefined
// Native clipboard images arrive with a fresh timestamped filename on every paste, so identical
Expand Down Expand Up @@ -138,13 +75,11 @@ export function createComposerAttachments(
target.prompt.set([...target.prompt.current(), attachment], target.cursor)
return true
}
const addAttachments = async (files: File[], toast = true, target = capture()) => {
const found = await files.reduce(async (result, file) => {
const addAttachments = async (files: File[], target = capture()) => {
return files.reduce(async (result, file) => {
const previous = await result
return (await add(file, false, target)) || previous
return (await add(file, target)) || previous
}, Promise.resolve(false))
if (!found && files.length > 0 && toast) input.warn()
return found
}
const handlePaste = async (event: ClipboardEvent) => {
const clipboardData = event.clipboardData
Expand All @@ -159,13 +94,13 @@ export function createComposerAttachments(
return file ? [file] : []
})
if (files.length > 0) {
await addAttachments(files, true, target)
await addAttachments(files, target)
return
}
const plainText = clipboardData.getData("text/plain") ?? ""
if (input.readClipboardImage && !plainText) {
const file = await input.readClipboardImage()
if (file && (await add(file, true, target, true))) return
if (file && (await add(file, target, true))) return
}
if (!plainText) return
const text = plainText.includes("\r") ? plainText.replace(/\r\n?/g, "\n") : plainText
Expand Down Expand Up @@ -223,9 +158,7 @@ export function createComposerAttachments(
fallback()
return
}
void input
.picker({ defaultPath: input.directory(), multiple: true, accept: accepted }, (file) => add(file))
.catch(input.onError)
void input.picker({ defaultPath: input.directory(), multiple: true }, (file) => add(file)).catch(input.onError)
},
}
}
Expand All @@ -249,6 +182,8 @@ const textMimes = new Set([
"application/yaml",
])

// Text-like files normalize to text/plain so the server inlines their content; every other
// file keeps a binary type and is delivered to the model by path or as native media.
async function attachmentMime(file: File) {
const type = file.type.split(";", 1)[0]?.trim().toLowerCase() ?? ""
if (imageMimes.has(type) || type === "application/pdf") return type
Expand All @@ -259,10 +194,11 @@ async function attachmentMime(file: File) {
if (type.startsWith("text/") || textMimes.has(type) || type.endsWith("+json") || type.endsWith("+xml")) {
return "text/plain"
}
const binary = type || "application/octet-stream"
const bytes = new Uint8Array(await file.slice(0, 4096).arrayBuffer())
if (bytes.some((byte) => byte === 0)) return
if (bytes.some((byte) => byte === 0)) return binary
const control = bytes.filter((byte) => byte < 9 || (byte > 13 && byte < 32)).length
if (bytes.length > 0 && control / bytes.length > 0.3) return
if (bytes.length > 0 && control / bytes.length > 0.3) return binary
return "text/plain"
}

Expand Down
70 changes: 70 additions & 0 deletions packages/app/src/composer/attachments/deliver.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import type { Accessor } from "solid-js"
import { blobBytes, blobDataUrl } from "@/runtime/persistence/drafts"
import { useServer } from "@/runtime/server/current"
import { useServerSDK } from "@/runtime/server/client"
import { useWorkspaceLocation } from "@/workspaces/location"
import type { ComposerControls } from "../adapter"
import type { ImageAttachmentPart } from "../state"

// Where a prompt is headed: the model that reads it and the server that runs its tools.
export type AttachmentDestination = {
/** Input modalities the selected model reads natively. */
input: { image: boolean; pdf: boolean }
/** The server shares the client's filesystem, so an attachment's source path resolves as-is. */
local: boolean
/** Copies a file into the server's temporary directory and returns its absolute path there. */
upload: (file: { name: string; data: Uint8Array }) => Promise<string>
}

export type DeliveredAttachment =
| { type: "inline"; attachment: ImageAttachmentPart; dataUrl: string }
| { type: "path"; attachment: ImageAttachmentPart; path: string }

// An attachment travels inline when the model reads its bytes natively. Anything else reaches
// the model as a path on the server, which its tools can open, instead of being rejected.
export function deliverAttachments(attachments: ImageAttachmentPart[], destination: AttachmentDestination) {
return Promise.all(attachments.map((attachment) => deliver(attachment, destination)))
}

async function deliver(
attachment: ImageAttachmentPart,
destination: AttachmentDestination,
): Promise<DeliveredAttachment> {
if (native(attachment.mime, destination.input)) {
return { type: "inline", attachment, dataUrl: await blobDataUrl(attachment.blob, attachment.mime) }
}
if (destination.local && attachment.sourcePath) return { type: "path", attachment, path: attachment.sourcePath }
const path = await destination.upload({ name: attachment.filename, data: await blobBytes(attachment.blob) })
return { type: "path", attachment, path }
}

const imageMimes = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"])

// Mirrors the attachment kinds the server forwards to the model as message content.
function native(mime: string, input: AttachmentDestination["input"]) {
if (mime === "text/plain") return true
if (imageMimes.has(mime)) return input.image
if (mime === "application/pdf") return input.pdf
return false
}

export function useAttachmentDestination(controls: Accessor<ComposerControls>) {
const server = useServer()
const sdk = useServerSDK()
const location = useWorkspaceLocation()
return (): AttachmentDestination => ({
input: controls().model.selection.current()?.capabilities.input ?? { image: false, pdf: false },
local: server.isLocal,
upload: async (file) => {
const info = await sdk.api.server.info()
// One directory per upload keeps the original filename without collisions; the server
// normalizes the separators and returns the resolved path.
const written = await sdk.api.file.write({
location: { directory: location().directory },
path: `${info.paths.tmp}/uploads/${crypto.randomUUID()}/${file.name}`,
payload: file.data,
})
return written.data.path
},
})
}
20 changes: 20 additions & 0 deletions packages/app/src/composer/comment-note.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,13 @@ export type PromptComment = {
origin?: "review" | "file"
}

/** An attachment the model receives as a path on the server rather than inline bytes. */
export type PromptAttachmentReference = {
name: string
mime: string
path: string
}

function selection(selection: unknown) {
if (!selection || typeof selection !== "object") return undefined
const startLine = Number((selection as FileSelection).startLine)
Expand Down Expand Up @@ -58,8 +65,17 @@ export function readPromptPresentation(value: unknown) {
const displayText = (value as { displayText?: unknown }).displayText
const comments = (value as { comments?: unknown }).comments
if (typeof displayText !== "string" || !Array.isArray(comments)) return
const attachments = (value as { attachments?: unknown }).attachments
return {
displayText,
attachments: (Array.isArray(attachments) ? attachments : []).flatMap((item): PromptAttachmentReference[] => {
if (!item || typeof item !== "object") return []
const name = (item as { name?: unknown }).name
const mime = (item as { mime?: unknown }).mime
const path = (item as { path?: unknown }).path
if (typeof name !== "string" || typeof mime !== "string" || typeof path !== "string") return []
return [{ name, mime, path }]
}),
comments: comments.flatMap((item): PromptComment[] => {
if (!item || typeof item !== "object") return []
const path = (item as { path?: unknown }).path
Expand All @@ -80,6 +96,10 @@ export function readPromptPresentation(value: unknown) {
}
}

export function formatAttachmentReference(input: PromptAttachmentReference) {
return `Attached file: \`${input.path}\``
}

export function formatCommentNote(input: { path: string; selection?: FileSelection; comment: string }) {
const start = input.selection ? Math.min(input.selection.startLine, input.selection.endLine) : undefined
const end = input.selection ? Math.max(input.selection.startLine, input.selection.endLine) : undefined
Expand Down
2 changes: 1 addition & 1 deletion packages/app/src/composer/composer.stories.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ function ComposerStory(props: {
? buildPromptRequest({
prompt: draft.prompt,
context: draft.context.items,
images: [],
attachments: [],
text: value,
sessionDirectory: "C:/repo",
})
Expand Down
1 change: 0 additions & 1 deletion packages/app/src/composer/editor/editor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ export function ComposerEditor(props: ComposerEditorProps) {
ref={props.controller.setFileInput}
type="file"
multiple
accept="image/png,image/jpeg,image/gif,image/webp,application/pdf,text/*,application/json,application/ld+json,application/toml,application/x-toml,application/x-yaml,application/xml,application/yaml,.c,.cc,.cjs,.conf,.cpp,.css,.csv,.cts,.env,.go,.gql,.graphql,.h,.hh,.hpp,.htm,.html,.ini,.java,.js,.json,.jsx,.log,.md,.mdx,.mjs,.mts,.py,.rb,.rs,.sass,.scss,.sh,.sql,.toml,.ts,.tsx,.txt,.xml,.yaml,.yml,.zsh"
class="hidden"
onChange={(event) => {
const list = event.currentTarget.files
Expand Down
7 changes: 2 additions & 5 deletions packages/app/src/composer/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import type { PromptHistoryComment } from "./history/entry"
import { createComposerHistory } from "./history/store"
import { composerPlaceholder } from "./placeholder"
import { createComposerSubmit } from "./submit"
import { useAttachmentDestination } from "./attachments/deliver"

export type ComposerModel = ComposerEditorModel & {
readonly model: ComposerControls["model"]
Expand Down Expand Up @@ -265,6 +266,7 @@ export function createComposerModel(adapter: ComposerAdapter, options?: { queue?
resetHistory: () => controller.resetHistory(),
setMode: (next) => controller.dispatch({ type: next === "shell" ? "mode.shell" : "mode.normal" }),
closePopover: () => controller.dispatch({ type: "popover.close" }),
destination: useAttachmentDestination(adapter.controls),
delivery: (alternate) => {
const queue = options?.queue
if (!queue) return "steer"
Expand Down Expand Up @@ -339,11 +341,6 @@ export function createComposerModel(adapter: ComposerAdapter, options?: { queue?
picker: platform.openAttachmentPickerDialog,
directory: () => sdk().directory,
isDialogActive: () => !!dialog.active,
warn: () =>
showToast({
title: language.t("prompt.toast.pasteUnsupported.title"),
description: language.t("prompt.toast.pasteUnsupported.description"),
}),
duplicate: () => showToast({ title: language.t("prompt.toast.attachmentDuplicate.title") }),
onError: (error) =>
showToast({
Expand Down
Loading
Loading