Skip to content

Include tools changes in existing turn analytics #50943

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aibrahim-oai wants to merge 6 commits into from
Closed

Include tools changes in existing turn analytics #50943

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c8023ab
Count model-visible tools changes across inference requests
aibrahim-oai Oct 4, 2026
a01c808
Keep tools metric test focused on cache changes, reuse standard telem…
aibrahim-oai Oct 4, 2026
696bba0
Send tools-change counts through existing backend turn telemetry
aibrahim-oai Oct 4, 2026
138746f
Exclude reused compaction streams from the tools change baseline
aibrahim-oai Oct 4, 2026
5c5a5f4
Count all prompt tool changes at the common inference entry point
aibrahim-oai Oct 4, 2026
cdf5e8f
Release tools baseline lock before comparison and turn profiling
aibrahim-oai Oct 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Keep tools metric test focused on cache changes, reuse standard telem… 
…etry tags
  • Loading branch information
@aibrahim-oai
aibrahim-oai committed Oct 4, 2026
commit a01c808bcb171ecf20683ad167de223b42687f1a
132 changes: 36 additions & 96 deletions codex-rs/core/src/client_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,42 +307,11 @@ fn test_session_telemetry() -> SessionTelemetry {
)
}

// Exercise both wire formats over HTTP, including a new turn and a transport reset.
// A tool's output schema stays local, whereas its description, parameters and order
// form the model-visible cache prefix.
#[tokio::test]
async fn inference_tools_metric_tracks_wire_changes_across_turns_by_originator()
-> anyhow::Result<()> {
// The comparison must use the wire representation, not ToolSpec's local-only fields.
#[test]
fn inference_tools_counter_counts_only_cache_relevant_changes() -> anyhow::Result<()> {
for use_responses_lite in [false, true] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/responses"))
.respond_with(
ResponseTemplate::new(/*status*/ 200)
.insert_header("content-type", "text/event-stream")
.set_body_string(
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"output\":[]}}\n\n",
),
)
.expect(/*requests*/ 8)
.mount(&server)
.await;
let client = ModelClient::new(
/*auth_manager*/ None,
AgentIdentityAuthPolicy::JwtOnly,
ThreadId::new(),
create_oss_provider_with_base_url(&format!("{}/v1", server.uri()), WireApi::Responses),
SessionSource::Cli,
"test-originator".to_string(),
/*model_verbosity*/ None,
/*content_item_kinds_enabled*/ true,
/*enable_request_compression*/ false,
/*include_timing_metrics*/ false,
/*beta_features_header*/ None,
/*concurrent_reasoning_summaries_enabled*/ false,
/*attestation_provider*/ None,
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
);
let client = test_model_client(SessionSource::Cli);
let metrics = MetricsClient::new(
MetricsConfig::in_memory(
"test",
Expand Down Expand Up @@ -378,51 +347,41 @@ async fn inference_tools_metric_tracks_wire_changes_across_turns_by_originator()
local_change.output_schema = Some(json!({"type": "object"}));
let mut schema_change = b.clone();
schema_change.parameters = JsonSchema::string(Some("Changed parameters".into()));
let tool_lists = [
vec![a.clone()], // baseline, no event
vec![local_change.clone()], // not in the request, no event
vec![local_change.clone(), b.clone()], // add
vec![local_change.clone(), b.clone()], // retry, no event
vec![b.clone(), a.clone()], // reorder
vec![schema_change.clone(), a.clone()], // schema changes, same tool names
vec![], // remove all
vec![], // retry, no event
];
let mut turn = client.new_session();
for (index, tools) in tool_lists.into_iter().enumerate() {
for (index, tools) in [
vec![a.clone()], // baseline
vec![local_change.clone()], // same on the wire
vec![local_change.clone(), b.clone()], // add
vec![local_change.clone(), b.clone()], // unchanged/retry
vec![b, a.clone()], // reorder
vec![schema_change, a], // schema
vec![], // remove all
vec![], // unchanged/retry
]
.into_iter()
.enumerate()
{
if index == 4 {
turn.try_switch_fallback_transport(&telemetry, &model_info);
drop(turn);
// New ModelClient clone/new turn must retain the prior comparison baseline.
turn = client.clone().new_session();
}
let prompt = Prompt {
tools: tools
.into_iter()
.map(ToolSpec::Function)
.collect::<Vec<_>>()
.into(),
base_instructions: BaseInstructions {
text: format!("Instructions may change: {index}"),
provenance: None,
let request = turn.client.build_responses_request(
&Prompt {
tools: tools
.into_iter()
.map(ToolSpec::Function)
.collect::<Vec<_>>()
.into(),
..Default::default()
},
..Default::default()
};
let mut stream = turn
.stream(
&prompt,
&model_info,
&telemetry,
/*effort*/ None,
codex_protocol::config_types::ReasoningSummary::None,
/*service_tier*/ None,
&metadata,
&InferenceTraceContext::disabled(),
)
.await?;
while let Some(event) = stream.next().await {
event?;
}
&model_info,
/*effort*/ None,
codex_protocol::config_types::ReasoningSummary::None,
/*service_tier*/ None,
&metadata,
)?;
turn.client.record_inference_tools(&request, &telemetry);
}
let snapshot = metrics.snapshot()?;
let recorded = snapshot
Expand All @@ -433,32 +392,13 @@ async fn inference_tools_metric_tracks_wire_changes_across_turns_by_originator()
let AggregatedMetrics::U64(MetricData::Sum(sum)) = recorded.data() else {
panic!("expected unsigned counter");
};
let points = sum
.data_points()
.map(|point| {
(
point.value(),
point
.attributes()
.map(|kv| (kv.key.as_str().to_string(), kv.value.as_str().to_string()))
.collect::<BTreeMap<_, _>>(),
)
})
.collect::<Vec<_>>();
assert_eq!(
points,
vec![(
4,
BTreeMap::from([
("app.version".into(), env!("CARGO_PKG_VERSION").into()),
("model".into(), "gpt-test".into()),
("originator".into(), "test-originator".into()),
("session_source".into(), "cli".into()),
])
)],
sum.data_points()
.map(|point| point.value())
.collect::<Vec<_>>(),
vec![4],
"use_responses_lite={use_responses_lite}"
);
server.verify().await;
}
Ok(())
}
Expand Down
Loading