Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/client/src/effect/api/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2344,6 +2344,7 @@ export type WebsearchProvidersOperation<E = never> = (
export type WebsearchQueryInput = {
readonly location?: { readonly directory?: string | undefined } | undefined
readonly query: string
readonly category?: WebSearch.Category | undefined
readonly providerID?: WebSearch.ID | undefined
}
export type WebsearchQueryOutput = { readonly location: Location.PublicRef; readonly data: WebSearch.Response }
Expand Down
2 changes: 1 addition & 1 deletion packages/client/src/effect/generated/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1542,7 +1542,7 @@ const EndpointWebsearchQuery = (raw: RawClient["server.websearch"]) => (input: W
preserveEffect<WebsearchQueryOutput>()(
raw["websearch.query"]({
query: { location: input["location"] },
payload: { query: input["query"], providerID: input["providerID"] },
payload: { query: input["query"], category: input["category"], providerID: input["providerID"] },
}).pipe(Effect.mapError(mapClientError)),
)

Expand Down
2 changes: 1 addition & 1 deletion packages/client/src/promise/generated/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2162,7 +2162,7 @@ export function make(options: ClientOptions) {
method: "POST",
path: `/api/websearch`,
query: { location: input["location"] },
body: { query: input["query"], providerID: input["providerID"] },
body: { query: input["query"], category: input["category"], providerID: input["providerID"] },
successStatus: 200,
declaredStatuses: [400, 401, 503],
empty: false,
Expand Down
13 changes: 11 additions & 2 deletions packages/client/src/promise/generated/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6553,8 +6553,17 @@ export type WebsearchProvidersOutput = { location: LocationPublicRef; data: Arra

export type WebsearchQueryInput = {
readonly location?: { readonly location?: { readonly directory?: string | undefined } | undefined }["location"]
readonly query: { readonly query: string; readonly providerID?: string }["query"]
readonly providerID?: { readonly query: string; readonly providerID?: string }["providerID"]
readonly query: { readonly query: string; readonly category?: "developer"; readonly providerID?: string }["query"]
readonly category?: {
readonly query: string
readonly category?: "developer"
readonly providerID?: string
}["category"]
readonly providerID?: {
readonly query: string
readonly category?: "developer"
readonly providerID?: string
}["providerID"]
}

export type WebsearchQueryOutput = {
Expand Down
2 changes: 2 additions & 0 deletions packages/core/src/plugin/host.ts
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,7 @@ export const make = Effect.fn("PluginHost.make")(function* (
response(
websearch.query({
query: input.query,
category: input.category,
providerID: input.providerID === undefined ? undefined : WebSearch.ID.make(input.providerID),
}),
),
Expand All @@ -493,6 +494,7 @@ export const make = Effect.fn("PluginHost.make")(function* (
editor.add({
id: WebSearch.ID.make(definition.id),
name: definition.name,
categories: definition.categories,
execute: definition.execute,
}),
default: {
Expand Down
65 changes: 59 additions & 6 deletions packages/core/src/plugin/websearch/firecrawl.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
export * as WebSearchFirecrawl from "./firecrawl.js"

import { define } from "@opencode/plugin/effect/plugin"
import { Effect, Option, Schema, Scope } from "effect"
import { HttpClient } from "effect/unstable/http"
import { Duration, Effect, Option, Schema, Scope } from "effect"
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
import { App } from "../../app.js"
import { collectBoundedResponseBody } from "../../tool/http-body.js"
import { WebSearchMcp } from "./mcp.js"

export const endpoint = "https://mcp.firecrawl.dev/v2/mcp"
// The developer index is on the REST API because the MCP tool for it is hidden from keyless sessions.
export const developerEndpoint = "https://api.firecrawl.dev/v2/search/developer"

const McpInput = Schema.Struct({
query: Schema.String,
Expand All @@ -33,6 +36,25 @@ const SearchResponse = Schema.fromJsonString(
)
const decodeSearchResponse = Schema.decodeUnknownOption(SearchResponse)

const DeveloperRequest = Schema.Struct({
query: Schema.String,
k: Schema.Number,
passages: Schema.Number,
})

const DeveloperResponse = Schema.fromJsonString(
Schema.Struct({
results: Schema.Array(
Schema.Struct({
url: Schema.String,
title: Schema.NullOr(Schema.String).pipe(Schema.optional),
passages: Schema.Array(Schema.Struct({ text: Schema.String })).pipe(Schema.optional),
}),
),
}),
)
const decodeDeveloperResponse = Schema.decodeUnknownEffect(DeveloperResponse)

export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
id: "opencode.websearch.firecrawl",
effect: Effect.fn("WebSearchFirecrawl.Plugin")(function* (ctx) {
Expand All @@ -52,20 +74,23 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
editor.add({
id: "firecrawl",
name: "Firecrawl",
categories: ["developer"],
execute: (input) =>
Effect.gen(function* () {
const connection = yield* ctx.integration.connection.active("firecrawl")
const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined
const headers = {
"User-Agent": App.useragent(ctx.app),
...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}),
}
if (input.category === "developer") return yield* developerSearch(http, input.query, headers)
const result = yield* WebSearchMcp.call(
http,
endpoint,
"firecrawl_search",
{ input: McpInput, output: McpOutput },
{ query: input.query, limit: 8 },
{
"User-Agent": App.useragent(ctx.app),
...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}),
},
headers,
)
const content = result?.content.find((item) => item.text)
const response = content ? Option.getOrUndefined(decodeSearchResponse(content.text)) : undefined
Expand All @@ -82,3 +107,31 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
})
}),
})

const developerSearch = (http: HttpClient.HttpClient, query: string, headers: Record<string, string>) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.post(developerEndpoint).pipe(
HttpClientRequest.acceptJson,
HttpClientRequest.setHeaders(headers),
HttpClientRequest.schemaBodyJson(DeveloperRequest)({ query, k: 8, passages: 3 }),
)
const response = yield* HttpClient.withScope(HttpClient.filterStatusOk(http)).execute(request)
const body = yield* collectBoundedResponseBody(
response,
WebSearchMcp.MAX_RESPONSE_BYTES,
() => new Error(`Firecrawl developer search response exceeded ${WebSearchMcp.MAX_RESPONSE_BYTES} bytes`),
)
const decoded = yield* decodeDeveloperResponse(body.toString("utf8"))
return decoded.results.map((item) => ({
url: item.url,
...(item.title ? { title: item.title } : {}),
...(item.passages?.length ? { content: item.passages.map((passage) => passage.text).join("\n\n") } : {}),
time: {},
}))
}).pipe(
Effect.scoped,
Effect.timeoutOrElse({
duration: Duration.seconds(25),
orElse: () => Effect.fail(new Error("Firecrawl developer search request timed out")),
}),
)
7 changes: 6 additions & 1 deletion packages/core/src/tool/plugin/websearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,15 @@ const httpErrors = new Map([

export const description = `Search the web using the user's selected search integration. Use this for current information beyond knowledge cutoff.

The current year is ${new Date().getFullYear()}. Use this year when searching for recent information or current events.`
The current year is ${new Date().getFullYear()}. Use this year when searching for recent information or current events.

Set category to "developer" for programming questions, such as how a library or API behaves, what an error message means, or whether a bug was fixed. Developer searches prefer an index of repositories, GitHub issues, merged pull requests, READMEs, and documentation, and return the matched passages.`

export const Input = Schema.Struct({
query: Schema.String.annotate({ description: "Websearch query" }),
category: WebSearch.Category.pipe(Schema.optional).annotate({
description: 'Set to "developer" for programming questions',
}),
})

const Output = Schema.Struct({
Expand Down
27 changes: 20 additions & 7 deletions packages/core/src/websearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ export type Provider = WebSearch.Provider

export { Event } from "@opencode/schema/websearch"

export const Category = WebSearch.Category
export type Category = WebSearch.Category

export const Input = WebSearch.Input
export type Input = WebSearch.Input
export type ProviderInput = WebSearch.ProviderInput
Expand All @@ -33,6 +36,8 @@ export const Selection = Schema.Union([ID, Schema.Literal("random"), Schema.Lite
export type Selection = typeof Selection.Type

export interface ProviderImplementation extends Provider {
/** Categories the provider searches natively. Providers ignore categories they do not list. */
readonly categories?: readonly Category[]
readonly execute: (input: ProviderInput) => Effect.Effect<readonly Result[], unknown>
}

Expand Down Expand Up @@ -130,17 +135,21 @@ const layer = Layer.effect(
return Option.getOrUndefined(decoded)
})

const randomProvider = (now: number, affinity: { provider?: ID }, attempted?: Set<ID>) => {
const randomProvider = (now: number, affinity: { provider?: ID }, attempted?: Set<ID>, category?: Category) => {
const providers = state.get().providers
cooldowns.forEach((cooldown, id) => {
if (cooldown.until <= now || !providers.has(id)) cooldowns.delete(id)
})
const current = affinity.provider === undefined ? undefined : providers.get(affinity.provider)
if (current && !cooldowns.has(current.id) && !attempted?.has(current.id)) return current
const available = Array.from(providers.values()).filter(
(provider) => !cooldowns.has(provider.id) && !attempted?.has(provider.id),
)
const provider = available[Math.floor(Math.random() * available.length)]
const current = available.find((provider) => provider.id === affinity.provider)
// Category queries prefer a provider that searches the category natively without moving the
// session's affinity, so general queries keep their provider.
const capable = category ? available.filter((provider) => provider.categories?.includes(category)) : []
if (capable.length) return capable.find((provider) => provider === current) ?? pick(capable)
if (current) return current
const provider = pick(available)
if (provider) affinity.provider = provider.id
return provider
}
Expand Down Expand Up @@ -184,7 +193,7 @@ const layer = Layer.effect(
const affinity = preferred.get(options?.sessionID) ?? { provider: undefined }
if (choice === "random") {
preferred.set(options?.sessionID, affinity)
provider = randomProvider(yield* Clock.currentTimeMillis, affinity) ?? provider
provider = randomProvider(yield* Clock.currentTimeMillis, affinity, undefined, input.category) ?? provider
}
const attempted = new Set<ID>()
while (true) {
Expand All @@ -193,7 +202,7 @@ const layer = Layer.effect(
if (!cooldown || cooldown.until <= (yield* Clock.currentTimeMillis)) {
attempted.add(provider.id)
const result = yield* provider
.execute({ query: input.query })
.execute({ query: input.query, category: input.category })
.pipe(Effect.flatMap(decodeResults), Effect.result)
if (result._tag === "Success") return new Response({ providerID: provider.id, results: result.success })
const cause = result.failure
Expand All @@ -204,14 +213,18 @@ const layer = Layer.effect(
cooldown = { until: now + cooldownMillis(cause.response.headers["retry-after"], now), error }
cooldowns.set(provider.id, cooldown)
}
provider = randomProvider(yield* Clock.currentTimeMillis, affinity, attempted)
provider = randomProvider(yield* Clock.currentTimeMillis, affinity, attempted, input.category)
if (!provider) return yield* cooldown.error
}
}),
})
}),
)

function pick<T>(items: readonly T[]) {
return items[Math.floor(Math.random() * items.length)]
}

function cooldownMillis(value: string | undefined, now: number) {
if (!value?.trim()) return 60_000
const seconds = Number(value)
Expand Down
53 changes: 53 additions & 0 deletions packages/core/test/plugin/websearch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,59 @@ describe("built-in web search providers", () => {
}),
)

it.effect("searches the Firecrawl developer index for developer queries", () =>
Effect.gen(function* () {
resetWebSearchFixture(
JSON.stringify({
success: true,
results: [
{
id: "issue:effect-ts/effect#1234",
type: "issue",
url: "https://github.com/effect-ts/effect/issues/1234",
title: "Retrying HttpClient requests",
passages: [{ text: "Use `Effect.retry` with a Schedule." }, { text: "Retries apply to 429 responses." }],
},
{ id: "doc:effect.website/retrying", type: "doc", url: "https://effect.website/docs/retrying" },
],
coverage: { doc: "ok", issue: "ok", pull_request: "ok", readme: "ok" },
reranked: true,
}),
)
const integrations = yield* Integration.Service
const websearch = yield* WebSearch.Service
yield* WebSearchFirecrawl.Plugin.effect(
host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }),
)
yield* websearch.select("random")
const query = { query: "effect retry", category: "developer" as const }

expect(yield* websearch.query(query)).toEqual(
new WebSearch.Response({
providerID: WebSearch.ID.make("firecrawl"),
results: [
{
url: "https://github.com/effect-ts/effect/issues/1234",
title: "Retrying HttpClient requests",
content: "Use `Effect.retry` with a Schedule.\n\nRetries apply to 429 responses.",
time: {},
},
{ url: "https://effect.website/docs/retrying", time: {} },
],
}),
)
expect(requests[0]).toMatchObject({
url: WebSearchFirecrawl.developerEndpoint,
body: { query: "effect retry", k: 8, passages: 3 },
})
expect(requests[0]?.headers.authorization).toBeUndefined()

yield* integrations.connection.key({ integrationID: Integration.ID.make("firecrawl"), key: "fc-secret" })
yield* websearch.query(query)
expect(requests[1]).toMatchObject({ headers: { authorization: "Bearer fc-secret" } })
}),
)

it.effect("registers Exa with its MCP schema", () =>
Effect.gen(function* () {
const integrations = yield* Integration.Service
Expand Down
23 changes: 23 additions & 0 deletions packages/core/test/tool-websearch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,29 @@ describe("WebSearchTool registration", () => {
}),
)

it.effect("forwards the search category to WebSearch", () =>
Effect.gen(function* () {
const fixture = yield* setup
yield* fixture.websearch.select(WebSearch.ID.make("exa"))

expect(
yield* executeTool(fixture.registry, {
sessionID,
...toolIdentity,
call: {
type: "tool-call",
id: "call-developer",
name: "websearch",
input: { query: "effect retry", category: "developer" },
},
}),
).toMatchObject({ status: "completed" })
expect(fixture.websearch.queries).toEqual([
{ query: "effect retry", category: "developer", providerID: undefined },
])
}),
)

it.effect("keeps normalized results in structured output", () =>
Effect.gen(function* () {
const fixture = yield* setup
Expand Down
Loading
Loading