1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 sync::{
4 Arc, Mutex,
5 atomic::{AtomicBool, AtomicU64, Ordering},
6 },
7};
8
9use cookie_agent_protocol::{
10 DelegateRequestPayload, DelegatedContextRole, DelegatedContextTurn, InvocationId,
11 PersistedAssistantPart, PersistedToolResult as ToolResult, ProducerId, ProducerOwner, RunId,
12 RunStartParams, SafeToolError, SessionId, SessionOrigin, SessionStatus, ToolCallId,
13 ToolCallTermination, ToolTerminationOutcome,
14};
15use serde_json::Value;
16use tokio::{sync::oneshot, task::JoinHandle};
17use tokio_util::sync::CancellationToken;
18
19use super::{
20 Engine, EngineError, Event, SessionCommand,
21 admission::{AdmissionGuard, InflightDelegation},
22 handles::SubagentScope,
23 helpers::{invocation_id, safe_code, safe_display, safe_error, session_depth},
24 producers::ProducerAuthority,
25};
26use crate::{
27 EngineHistoryView,
28 delegation_api::{DelegateAwait, DelegateHandle, DelegateInvocation},
29 delegation_events::{self, DelegationEventError},
30 policy::{self, FrozenRunPolicy, freeze_delegated_agent_policy, resolve_agent},
31 session,
32};
33
34pub(super) const DELEGATION_RECOVERY_SETTLE_BOUND: std::time::Duration =
42 std::time::Duration::from_secs(5);
43
44pub(crate) struct DelegationRuntimeState {
46 pub(super) inflight: Mutex<HashMap<InvocationId, HashMap<u64, InflightDelegation>>>,
47 pub(super) by_session: Mutex<HashMap<SessionId, DelegationRecord>>,
48 pub(crate) queue: Mutex<VecDeque<SessionId>>,
49 pub(crate) admission: tokio::sync::Mutex<()>,
50 pub(crate) reconciliation_running: AtomicBool,
51 pub(crate) reconciliation_requested: AtomicBool,
52 pub(crate) recovery_stale_producers: Mutex<Vec<(SessionId, InvocationId, ProducerId)>>,
53 pub(crate) next_admission_generation: AtomicU64,
54 pub(crate) admission_tasks: Mutex<Vec<JoinHandle<()>>>,
55 pub(crate) admission_blocking_tasks: Mutex<Vec<JoinHandle<()>>>,
56 pub(crate) admission_tasks_closing: AtomicBool,
57 pub(crate) recovery_waiters: Mutex<HashSet<(SessionId, RunId, ToolCallId)>>,
58 pub(crate) recovery_pending: Mutex<HashMap<SessionId, usize>>,
65 pub(crate) recovery_settled: tokio::sync::Notify,
68}
69
70impl Default for DelegationRuntimeState {
71 fn default() -> Self {
72 Self {
73 inflight: Mutex::new(HashMap::new()),
74 by_session: Mutex::new(HashMap::new()),
75 queue: Mutex::new(VecDeque::new()),
76 admission: tokio::sync::Mutex::new(()),
77 reconciliation_running: AtomicBool::new(false),
78 reconciliation_requested: AtomicBool::new(false),
79 recovery_stale_producers: Mutex::new(Vec::new()),
80 next_admission_generation: AtomicU64::new(1),
82 admission_tasks: Mutex::new(Vec::new()),
83 admission_blocking_tasks: Mutex::new(Vec::new()),
84 admission_tasks_closing: AtomicBool::new(false),
85 recovery_waiters: Mutex::new(HashSet::new()),
86 recovery_pending: Mutex::new(HashMap::new()),
87 recovery_settled: tokio::sync::Notify::new(),
88 }
89 }
90}
91
92impl Engine {
93 pub(crate) fn validate_resume_target(
94 &self,
95 parent_session_id: SessionId,
96 resume_session_id: SessionId,
97 agent_type: &cookie_agent_protocol::AgentId,
98 preset: Option<&str>,
99 ) -> Result<session::SessionProjection, EngineError> {
100 if resume_session_id == parent_session_id {
101 return Err(EngineError::ToolFailed(
102 "resume_session_id cannot reference the delegating session itself".into(),
103 ));
104 }
105 let mut cursor = parent_session_id;
106 let mut visited = std::collections::HashSet::new();
107 while visited.insert(cursor) {
108 let current = self.inner.store.get(cursor)?;
109 let SessionOrigin::Delegated {
110 parent_session_id, ..
111 } = current.meta.origin
112 else {
113 break;
114 };
115 if parent_session_id == resume_session_id {
116 return Err(EngineError::ToolFailed(
117 "resume_session_id cannot reference an ancestor session".into(),
118 ));
119 }
120 cursor = parent_session_id;
121 }
122 let resumed = self.inner.store.get(resume_session_id).map_err(|_| {
123 EngineError::ToolFailed(format!("resume session {resume_session_id} was not found"))
124 })?;
125 let direct_child = matches!(
126 resumed.meta.origin,
127 SessionOrigin::Delegated {
128 parent_session_id: owner,
129 ..
130 } if owner == parent_session_id
131 ) && self.inner.delegation_events.entries().iter().any(|entry| {
132 entry.reservation.parent_session_id == parent_session_id
133 && entry.reservation.child_session_id == resume_session_id
134 });
135 if !direct_child {
136 return Err(EngineError::ToolFailed(
137 "resume session is not a prior direct child of the delegating parent".into(),
138 ));
139 }
140 if &resumed.meta.creation_selection.agent != agent_type {
141 return Err(EngineError::ToolFailed(
142 "resume session agent does not match agent_type".into(),
143 ));
144 }
145 if resumed.meta.creation_selection.preset.as_deref() != preset {
146 return Err(EngineError::ToolFailed(
147 "resume target belongs to a different agent preset".into(),
148 ));
149 }
150 if let Some(record) = self
151 .inner
152 .delegation
153 .by_session
154 .lock()
155 .map_err(|_| EngineError::ActorStopped)?
156 .get(&resume_session_id)
157 .copied()
158 && matches!(
159 record.state,
160 DelegationState::Queued | DelegationState::Starting
161 )
162 {
163 return Err(EngineError::ToolFailed(format!(
164 "resume session already has in-flight delegation {} in {} state",
165 record.invocation_id,
166 if record.state == DelegationState::Queued {
167 "queued"
168 } else {
169 "starting"
170 }
171 )));
172 }
173 Ok(resumed)
174 }
175
176 async fn delegated_context_seed(
177 &self,
178 parent_session_id: SessionId,
179 ) -> Result<Vec<DelegatedContextTurn>, EngineError> {
180 let history = self
181 .get_history(parent_session_id, EngineHistoryView::Assembled)
182 .await?;
183 Ok(context_seed_from_history(history))
184 }
185
186 pub(super) fn is_delegate_call_result(
187 &self,
188 session: SessionId,
189 run: RunId,
190 tool_call_id: ToolCallId,
191 result: &ToolResult,
192 ) -> bool {
193 let Ok(parent) = self.inner.store.get(session) else {
194 return false;
195 };
196 if parent
197 .runs
198 .get(&run)
199 .and_then(|run| run.pending_calls.get(&tool_call_id))
200 .is_none_or(|name| name != "delegate_subagent")
201 {
202 return false;
203 }
204 self.inner
205 .delegation_events
206 .get(invocation_id(session, run, tool_call_id))
207 .is_some_and(|entry| {
208 entry.reservation.parent_session_id == session
209 && entry.reservation.parent_run_id == run
210 && entry.reservation.parent_tool_call_id == tool_call_id
211 && crate::delegation_api::delegate_result_matches_child(
212 result,
213 entry.reservation.child_session_id,
214 )
215 })
216 }
217
218 pub(super) async fn ensure_parent_link(
221 &self,
222 parent_session_id: SessionId,
223 parent_run_id: RunId,
224 parent_tool_call_id: ToolCallId,
225 child_session_id: SessionId,
226 ) -> Result<(), EngineError> {
227 self.request(parent_session_id, |reply| {
228 SessionCommand::EnsureToolCallLinked {
229 run: parent_run_id,
230 tool_call_id: parent_tool_call_id,
231 child_session_id,
232 reply,
233 }
234 })
235 .await
236 }
237
238 pub(super) async fn terminal_parent_delegate(
239 &self,
240 parent_session_id: SessionId,
241 parent_run_id: RunId,
242 parent_tool_call_id: ToolCallId,
243 ) -> Result<bool, EngineError> {
244 let parent = self.inner.store.get(parent_session_id)?;
245 let Some(run) = parent.runs.get(&parent_run_id) else {
246 return Ok(true);
247 };
248 if !matches!(
249 run.status,
250 SessionStatus::Cancelled | SessionStatus::Failed | SessionStatus::Completed
251 ) {
252 return Ok(false);
253 }
254 if run
255 .pending_calls
256 .get(&parent_tool_call_id)
257 .is_some_and(|tool| tool == "delegate_subagent")
258 {
259 let result = cancelled_delegate_result_with_reason(
260 None,
261 None,
262 "parent run was already terminal",
263 );
264 self.append(
265 parent_session_id,
266 Some(parent_run_id),
267 super::event_origin("engine:delegation"),
268 Event::ToolCallTerminated {
269 termination: ToolCallTermination {
270 tool_call_id: parent_tool_call_id,
271 owner: self.tool_call_owner(
272 parent_session_id,
273 parent_run_id,
274 parent_tool_call_id,
275 )?,
276 outcome: ToolTerminationOutcome::Cancelled,
277 result: Some(result),
278 error: Some(SafeToolError {
279 code: safe_code("parent_terminal"),
280 message: safe_error("parent run was already terminal"),
281 }),
282 },
283 },
284 )
285 .await?;
286 }
287 Ok(true)
288 }
289
290 pub async fn delegate_invoke(
293 &self,
294 invocation: DelegateInvocation,
295 ) -> Result<DelegateHandle, EngineError> {
296 let invocation_id = invocation_id(
297 invocation.parent_session_id,
298 invocation.parent_run_id,
299 invocation.parent_tool_call_id,
300 );
301 let mut admission =
302 AdmissionGuard::begin(self.inner.clone(), invocation_id, invocation.parent_run_id);
303 admission.set_parent(invocation.parent_session_id, invocation.parent_tool_call_id);
304 let (reply, receiver) = oneshot::channel();
307 let engine = self.clone();
308 let generation = admission.generation;
309 let Some(runtime) = tokio::runtime::Handle::try_current().ok() else {
310 return Err(EngineError::ActorStopped);
311 };
312 if !self.spawn_admission_task(&runtime, async move {
313 let result = engine
314 .delegate_invoke_admitted(invocation, invocation_id, generation)
315 .await;
316 let _ = reply.send(result);
317 }) {
318 return Err(EngineError::ActorStopped);
319 }
320 let result = receiver.await.map_err(|_| EngineError::ActorStopped)?;
321 if result.is_ok() {
322 admission.handoff();
323 } else {
324 admission.complete();
325 }
326 result
327 }
328
329 pub(super) async fn delegate_invoke_admitted(
330 &self,
331 invocation: DelegateInvocation,
332 invocation_id: InvocationId,
333 generation: u64,
334 ) -> Result<DelegateHandle, EngineError> {
335 let admission_guard = self.inner.delegation.admission.lock().await;
336 if invocation
337 .prompt
338 .starts_with(super::skills::RESERVED_STAGED_SKILL_PREFIX)
339 {
340 return Err(EngineError::ToolFailed(
341 "delegate prompt uses a reserved staged-skill prefix".into(),
342 ));
343 }
344 let staged_skill = self
345 .inner
346 .skills_runtime
347 .pending_forks
348 .lock()
349 .unwrap_or_else(|poisoned| poisoned.into_inner())
350 .get(&invocation.parent_tool_call_id)
351 .map(super::skills::PreparedSkillInvocation::staged_payload);
352 if invocation.resume_session_id.is_some() && invocation.inherit_context {
353 return Err(EngineError::ToolFailed(
354 "resume_session_id and inherit_context cannot both be set".into(),
355 ));
356 }
357 let resume_session_id = invocation
358 .resume_session_id
359 .as_deref()
360 .map(|reference| {
361 self.resolve_subagent_target(
362 invocation.parent_session_id,
363 reference,
364 SubagentScope::DirectChildren,
365 )
366 })
367 .transpose()?;
368 if let Some(existing) = self.delegation_event_get(invocation_id).await?
369 && existing.child_run_id.is_some()
370 {
371 validate_redelivery_mode(existing.request.background, invocation.background)?;
372 let request_matches = existing.reservation.parent_session_id
373 == invocation.parent_session_id
374 && existing.reservation.parent_run_id == invocation.parent_run_id
375 && existing.reservation.parent_tool_call_id == invocation.parent_tool_call_id
376 && existing.child_agent.agent == invocation.agent_type
377 && existing.request.description == invocation.description
378 && existing.request.prompt == invocation.prompt
379 && existing.request.resume_session_id == resume_session_id
380 && existing.request.inherit_context == invocation.inherit_context
381 && existing.request.staged_skill == staged_skill;
382 if !request_matches {
383 return Err(EngineError::ToolFailed(
384 "delegate redelivery does not match its durable reservation".into(),
385 ));
386 }
387 let registry_owns_invocation = self
388 .inner
389 .delegation
390 .by_session
391 .lock()
392 .map_err(|_| EngineError::ActorStopped)?
393 .get(&existing.reservation.child_session_id)
394 .is_some_and(|record| record.invocation_id == invocation_id);
395 if existing.run_attached || registry_owns_invocation {
396 if existing.request.background && registry_owns_invocation {
397 let producer_missing = self
398 .inner
399 .delegation
400 .by_session
401 .lock()
402 .map_err(|_| EngineError::ActorStopped)?
403 .get(&existing.reservation.child_session_id)
404 .is_some_and(|record| {
405 !record.notification_sent && record.producer_id.is_none()
406 });
407 if producer_missing {
408 let mut guard = self
409 .register_background_delegation_producer(
410 existing.reservation.parent_session_id,
411 invocation_id,
412 )
413 .await?;
414 let mut records = self
415 .inner
416 .delegation
417 .by_session
418 .lock()
419 .map_err(|_| EngineError::ActorStopped)?;
420 if let Some(record) = records
421 .get_mut(&existing.reservation.child_session_id)
422 .filter(|record| {
423 record.invocation_id == invocation_id
424 && record.producer_id.is_none()
425 })
426 {
427 record.producer_id = Some(guard.producer_id);
428 guard.handoff();
429 }
430 }
431 }
432 return Ok(DelegateHandle {
433 invocation_id,
434 child_session_id: existing.reservation.child_session_id,
435 child_run_id: existing.child_run_id,
436 });
437 }
438 }
439 let parent = self.inner.store.get(invocation.parent_session_id)?;
440 if parent
441 .runs
442 .get(&invocation.parent_run_id)
443 .is_some_and(|run| run.status == SessionStatus::Interrupted)
444 && self.delegation_event_get(invocation_id).await?.is_none()
445 {
446 return Err(EngineError::ToolFailed(
447 "delegate parent run is interrupted; use recovery".into(),
448 ));
449 }
450 let active_parent = self
451 .inner
452 .sessions
453 .active
454 .lock()
455 .unwrap_or_else(|poisoned| poisoned.into_inner())
456 .get(&invocation.parent_run_id)
457 .cloned()
458 .ok_or_else(|| EngineError::ToolFailed("delegate parent run is not active".into()))?;
459 let parent_policy = Arc::clone(&active_parent.policy);
460 let parent_delegation = parent_policy
461 .agent
462 .delegation
463 .as_ref()
464 .ok_or_else(|| EngineError::ToolFailed("delegation is disabled".into()))?;
465 if !parent_delegation.targets.contains(&invocation.agent_type)
466 || session_depth(&parent.meta.origin) >= parent_delegation.effective_depth_ceiling
467 {
468 return Err(EngineError::ToolFailed(
469 "delegate target or depth is not allowed".into(),
470 ));
471 }
472 let resume_child = resume_session_id
473 .map(|resume_session_id| {
474 self.validate_resume_target(
475 invocation.parent_session_id,
476 resume_session_id,
477 &invocation.agent_type,
478 parent_policy.preset.as_deref(),
479 )
480 })
481 .transpose()?;
482 let child_agent = resolve_agent(&parent_policy.registry, &invocation.agent_type)?;
483 if !child_agent.document.frontmatter.enabled
484 || !matches!(
485 child_agent.document.frontmatter.mode,
486 cookie_agent_config::AgentMode::Subagent | cookie_agent_config::AgentMode::All
487 )
488 {
489 return Err(EngineError::DisabledAgent(invocation.agent_type));
490 }
491 let fallback_index = active_parent.fallback_index.load(Ordering::Acquire) as usize;
492 let inherited = parent_policy.active_suffix(fallback_index);
493 let child_selection = child_agent
494 .resolved_fallback
495 .iter()
496 .filter_map(|fallback| match fallback {
497 crate::runtime_snapshot::ResolvedAgentFallback::Selection { selection, .. } => {
498 Some(selection)
499 }
500 crate::runtime_snapshot::ResolvedAgentFallback::ParentModel { .. } => None,
501 })
502 .find(|selection| {
503 parent_policy
504 .runtime
505 .models
506 .model(&selection.model)
507 .is_some_and(|model| {
508 model.model.status
509 == cookie_agent_models::compiler::CompiledModelStatus::Available
510 })
511 })
512 .cloned()
513 .or_else(|| {
514 if child_agent.resolved_fallback.is_empty() {
515 inherited.first().map(|binding| binding.selection.clone())
516 } else {
517 None
518 }
519 })
520 .ok_or_else(|| EngineError::ToolFailed("delegate child has no model suffix".into()))?;
521 let child_policy = freeze_delegated_child_policy(
522 child_agent,
523 &parent_policy,
524 &child_selection,
525 inherited,
526 parent_delegation.effective_depth_ceiling,
527 policy::ResultLimits {
528 tool_output_max_lines: self.inner.config.runtime.tool_output.max_lines,
529 tool_output_max_bytes: self.inner.config.runtime.tool_output.max_bytes,
530 },
531 )?;
532 let seeded_context = if invocation.inherit_context {
533 self.delegated_context_seed(invocation.parent_session_id)
534 .await?
535 } else {
536 Vec::new()
537 };
538 let proposed_title = super::titles::delegated_title(
539 &invocation.description,
540 self.inner.config.runtime.session_title.max_chars,
541 )?;
542 let title = resume_child
543 .as_ref()
544 .and_then(|child| child.meta.title.clone())
545 .unwrap_or(proposed_title);
546 let request = DelegateRequestPayload {
547 description: invocation.description,
548 prompt: invocation.prompt,
549 title,
550 resume_session_id,
551 inherit_context: invocation.inherit_context,
552 seeded_context,
553 background: invocation.background,
554 staged_skill,
555 };
556 let cache_strategies = policy::wire_cache_strategies(&child_policy.cache_strategies)?;
557 let fingerprint = delegation_events::delegation_request_fingerprint(
558 &child_policy.agent,
559 &child_policy.selected_suffix,
560 &cache_strategies,
561 &request,
562 )
563 .map_err(|()| EngineError::RuntimeCompileFailed)?;
564
565 let root_session_id = match parent.meta.origin {
566 SessionOrigin::Delegated {
567 root_session_id, ..
568 } => root_session_id,
569 _ => parent.meta.session_id,
570 };
571 let resumed_running = if resume_child
572 .as_ref()
573 .is_some_and(|child| child.status == SessionStatus::Running)
574 {
575 let record = self
576 .inner
577 .delegation
578 .by_session
579 .lock()
580 .map_err(|_| EngineError::ActorStopped)?
581 .get(
582 &resume_child
583 .as_ref()
584 .expect("checked resume child")
585 .meta
586 .session_id,
587 )
588 .copied()
589 .ok_or_else(|| {
590 EngineError::ToolFailed(
591 "running resume session has no active delegation record".into(),
592 )
593 })?;
594 let run_id = record.child_run_id.ok_or_else(|| {
595 EngineError::ToolFailed("running resume session has no active run".into())
596 })?;
597 Some((run_id, record))
598 } else {
599 None
600 };
601 let fresh_counts_slot =
602 invocation.background && !matches!(parent.meta.origin, SessionOrigin::Delegated { .. });
603 let counts_slot = resumed_running
604 .map(|(_, record)| record.counts_slot)
605 .unwrap_or(fresh_counts_slot);
606 let queued = resumed_running.is_none()
607 && fresh_counts_slot
608 && self.background_slot_unavailable(root_session_id)?;
609 if queued && self.background_queue_full(root_session_id)? {
610 return Err(EngineError::ToolFailed(format!(
611 "delegate admission denied: background queue is full for concurrency limit {}",
612 self.inner
613 .config
614 .runtime
615 .delegation
616 .max_concurrency
617 .unwrap_or(4)
618 )));
619 }
620 let mut producer_guard = if invocation.background {
621 Some(
622 self.register_background_delegation_producer(
623 invocation.parent_session_id,
624 invocation_id,
625 )
626 .await?,
627 )
628 } else {
629 None
630 };
631 let child = match self
632 .create_child(
633 invocation.parent_session_id,
634 invocation.parent_run_id,
635 invocation.parent_tool_call_id,
636 &invocation.agent_type,
637 child_policy,
638 cache_strategies,
639 fingerprint,
640 request,
641 Some((invocation_id, generation)),
642 )
643 .await
644 {
645 Ok(child) => child,
646 Err(error) => {
647 if is_delegation_event_append_failure(&error) {
648 let result = delegate_failure_result(
649 None,
650 None,
651 "delegate reservation event append failed",
652 );
653 self.resolve_delegate_failure_if_pending(
654 invocation.parent_session_id,
655 invocation.parent_run_id,
656 invocation.parent_tool_call_id,
657 result,
658 )
659 .await?;
660 }
661 return Err(error);
662 }
663 };
664 self.transfer_pending_skill_fork(invocation.parent_tool_call_id, child.session_id);
665 self.publish_admission_child(invocation_id, generation, child.session_id)?;
666 let entry = self
667 .delegation_event_get(invocation_id)
668 .await?
669 .ok_or_else(|| EngineError::ToolFailed("delegate reservation disappeared".into()))?;
670 if let Some(guard) = producer_guard.as_mut() {
671 let mut records = self
672 .inner
673 .delegation
674 .by_session
675 .lock()
676 .map_err(|_| EngineError::ActorStopped)?;
677 if let Some(record) = records.get_mut(&child.session_id).filter(|record| {
678 record.invocation_id == invocation_id && record.producer_id.is_none()
679 }) {
680 record.producer_id = Some(guard.producer_id);
681 guard.handoff();
682 }
683 }
684 #[cfg(test)]
685 let skill_fork_hook = if entry.request.staged_skill.is_some() {
686 self.inner
687 .test_hooks
688 .skill_fork_reservation_hook
689 .lock()
690 .expect("skill fork reservation hook lock poisoned")
691 .take()
692 } else {
693 None
694 };
695 #[cfg(test)]
696 if let Some(hook) = skill_fork_hook {
697 if let Some(reached) = hook
698 .reached
699 .lock()
700 .expect("skill fork reservation reached lock poisoned")
701 .take()
702 {
703 let _ = reached.send(());
704 }
705 hook.release.notified().await;
706 }
707 if resume_child.is_some() {
708 self.inner
709 .delegation
710 .queue
711 .lock()
712 .map_err(|_| EngineError::ActorStopped)?
713 .retain(|session_id| *session_id != child.session_id);
714 }
715 if let Some((child_run_id, previous_record)) = resumed_running {
716 if entry.child_run_id == Some(child_run_id)
717 && (entry.run_attached || previous_record.invocation_id == invocation_id)
718 {
719 return Ok(DelegateHandle {
720 invocation_id,
721 child_session_id: child.session_id,
722 child_run_id: Some(child_run_id),
723 });
724 }
725 let delegation_events = self.inner.delegation_events.clone();
726 self.spawn_admission_blocking(move || {
727 delegation_events.mark_run_attached(invocation_id, child_run_id)
728 })
729 .await?;
730 #[cfg(test)]
731 self.wait_for_resume_test_hook(&self.inner.test_hooks.resume_attachment_hook)
732 .await;
733 if let Err(error) = self.publish_resume_admission_target(
734 invocation_id,
735 generation,
736 child.session_id,
737 child_run_id,
738 previous_record,
739 ) {
740 let delegation_events = self.inner.delegation_events.clone();
741 self.spawn_admission_blocking(move || {
742 delegation_events.mark_finished(invocation_id, SessionStatus::Cancelled)
743 })
744 .await?;
745 return Err(error);
746 }
747 let handle = DelegateHandle {
748 invocation_id,
749 child_session_id: child.session_id,
750 child_run_id: Some(child_run_id),
751 };
752 let monitor_release = if invocation.background || counts_slot {
753 match self.spawn_background_monitor_gated(handle) {
754 Ok(release) => Some(release),
755 Err(error) => {
756 let delegation_events = self.inner.delegation_events.clone();
757 self.spawn_admission_blocking(move || {
758 delegation_events.mark_finished(invocation_id, SessionStatus::Cancelled)
759 })
760 .await?;
761 return Err(error);
762 }
763 }
764 } else {
765 None
766 };
767 #[cfg(test)]
768 if let Some(hook) = {
769 self.inner
770 .test_hooks
771 .resume_admission_hook
772 .lock()
773 .expect("resume admission hook lock poisoned")
774 .clone()
775 } {
776 if let Some(reached) = hook
777 .reached
778 .lock()
779 .expect("resume admission reached lock poisoned")
780 .take()
781 {
782 let _ = reached.send(());
783 }
784 hook.release.notified().await;
785 }
786 let admission = match self
787 .request(child.session_id, |reply| {
788 SessionCommand::AdmitDelegatedResume {
789 run: child_run_id,
790 input: entry.request.prompt.clone(),
791 reply,
792 }
793 })
794 .await
795 {
796 Ok(admission) => admission,
797 Err(error) => {
798 let delegation_events = self.inner.delegation_events.clone();
799 self.spawn_admission_blocking(move || {
800 delegation_events.mark_finished(invocation_id, SessionStatus::Cancelled)
801 })
802 .await?;
803 return Err(error);
804 }
805 };
806 if !admission.accepted {
807 let delegation_events = self.inner.delegation_events.clone();
808 self.spawn_admission_blocking(move || {
809 delegation_events.mark_finished(invocation_id, SessionStatus::Cancelled)
810 })
811 .await?;
812 return Err(EngineError::ToolFailed(
813 "resume session stopped before its prompt was admitted".into(),
814 ));
815 }
816 let Some(admission_seq) = admission.admission_seq else {
817 let delegation_events = self.inner.delegation_events.clone();
818 self.spawn_admission_blocking(move || {
819 delegation_events.mark_finished(invocation_id, SessionStatus::Cancelled)
820 })
821 .await?;
822 let _ = self.cancel_run_durably(
823 child_run_id,
824 Some("resume admission sequence was not published".into()),
825 );
826 return Err(EngineError::ToolFailed(
827 "resume admission sequence is missing".into(),
828 ));
829 };
830 if let Err(error) =
831 self.publish_resume_admission_seq(invocation_id, generation, admission_seq)
832 {
833 drop(monitor_release);
834 self.rollback_running_resume(
835 invocation_id,
836 child.session_id,
837 child_run_id,
838 admission_seq,
839 previous_record,
840 SessionStatus::Cancelled,
841 )
842 .await?;
843 return Err(error);
844 }
845 #[cfg(test)]
846 self.wait_for_resume_test_hook(&self.inner.test_hooks.resume_rollback_hook)
847 .await;
848 if !self.admission_generation_live(invocation_id, generation) {
849 drop(monitor_release);
850 self.rollback_running_resume(
851 invocation_id,
852 child.session_id,
853 child_run_id,
854 admission_seq,
855 previous_record,
856 SessionStatus::Cancelled,
857 )
858 .await?;
859 return Err(EngineError::ToolFailed(
860 "delegate admission was abandoned".into(),
861 ));
862 }
863 let registry_failed = {
864 match self.inner.delegation.by_session.lock() {
865 Ok(mut records) => {
866 let record = DelegationRecord {
867 parent_session_id: invocation.parent_session_id,
868 parent_run_id: invocation.parent_run_id,
869 parent_tool_call_id: invocation.parent_tool_call_id,
870 invocation_id,
871 root_session_id,
872 child_run_id: Some(child_run_id),
873 state: DelegationState::Running,
874 background: invocation.background,
875 counts_slot,
876 notification_sent: false,
877 producer_id: producer_guard.as_ref().map(|guard| guard.producer_id),
878 monitor_started: monitor_release.is_some(),
879 };
880 debug_assert!(record.background || record.producer_id.is_none());
881 records.insert(child.session_id, record);
882 false
883 }
884 Err(_) => true,
885 }
886 };
887 if registry_failed {
888 drop(monitor_release);
889 self.rollback_running_resume(
890 invocation_id,
891 child.session_id,
892 child_run_id,
893 admission_seq,
894 previous_record,
895 SessionStatus::Cancelled,
896 )
897 .await?;
898 return Err(EngineError::ActorStopped);
899 }
900 if let Some(release) = monitor_release
901 && release.send(()).is_err()
902 {
903 self.rollback_running_resume(
904 invocation_id,
905 child.session_id,
906 child_run_id,
907 admission_seq,
908 previous_record,
909 SessionStatus::Cancelled,
910 )
911 .await?;
912 return Err(EngineError::ActorStopped);
913 }
914 if !self.admission_generation_live(invocation_id, generation) {
915 self.rollback_running_resume(
916 invocation_id,
917 child.session_id,
918 child_run_id,
919 admission_seq,
920 previous_record,
921 SessionStatus::Cancelled,
922 )
923 .await?;
924 return Err(EngineError::ToolFailed(
925 "delegate admission was abandoned".into(),
926 ));
927 }
928 if let Some(guard) = producer_guard.as_mut() {
929 guard.handoff();
930 }
931 return Ok(handle);
932 }
933 let record = DelegationRecord {
934 parent_session_id: invocation.parent_session_id,
935 parent_run_id: invocation.parent_run_id,
936 parent_tool_call_id: invocation.parent_tool_call_id,
937 invocation_id,
938 root_session_id,
939 child_run_id: None,
940 state: if queued {
941 DelegationState::Queued
942 } else {
943 DelegationState::Starting
944 },
945 background: invocation.background,
946 counts_slot,
947 notification_sent: false,
948 producer_id: producer_guard.as_ref().map(|guard| guard.producer_id),
949 monitor_started: false,
950 };
951 debug_assert!(record.background || record.producer_id.is_none());
952 self.inner
953 .delegation
954 .by_session
955 .lock()
956 .map_err(|_| EngineError::ActorStopped)?
957 .insert(child.session_id, record);
958 if let Some(guard) = producer_guard.as_mut() {
959 guard.handoff();
960 }
961 if queued {
962 let position = {
963 let mut queue = self
964 .inner
965 .delegation
966 .queue
967 .lock()
968 .map_err(|_| EngineError::ActorStopped)?;
969 queue.push_back(child.session_id);
970 u32::try_from(queue.len()).ok()
971 };
972 if let Err(error) = self
973 .append(
974 invocation.parent_session_id,
975 Some(invocation.parent_run_id),
976 super::event_origin("engine:delegation"),
977 Event::DelegateQueued {
978 session_id: child.session_id,
979 position,
980 },
981 )
982 .await
983 {
984 self.inner
985 .delegation
986 .queue
987 .lock()
988 .map_err(|_| EngineError::ActorStopped)?
989 .retain(|session_id| *session_id != child.session_id);
990 let delegation_events = self.inner.delegation_events.clone();
991 let _ = self
992 .spawn_admission_blocking(move || {
993 delegation_events.mark_finished(invocation_id, SessionStatus::Failed)
994 })
995 .await;
996 let terminalized = self
997 .terminalize_child_without_run(
998 child.session_id,
999 SessionStatus::Failed,
1000 "delegate queue event append failed",
1001 )
1002 .await;
1003 if let Some(record) = self
1004 .inner
1005 .delegation
1006 .by_session
1007 .lock()
1008 .map_err(|_| EngineError::ActorStopped)?
1009 .get_mut(&child.session_id)
1010 {
1011 record.state = DelegationState::Finished(SessionStatus::Failed);
1012 }
1013 drop(admission_guard);
1014 if terminalized.is_ok() {
1015 self.finish_background_or_retry(child.session_id, invocation_id)
1016 .await;
1017 } else {
1018 let _ = self.start_queued_delegation(root_session_id).await;
1019 }
1020 return Err(error);
1021 }
1022 return Ok(DelegateHandle {
1023 invocation_id,
1024 child_session_id: child.session_id,
1025 child_run_id: None,
1026 });
1027 }
1028 let child_run_id = match self
1029 .ensure_delegate_run(&entry, Some((invocation_id, generation)))
1030 .await
1031 {
1032 Ok(run_id) => run_id,
1033 Err(error) => {
1034 if is_delegation_event_append_failure(&error) {
1035 let result = delegate_failure_result(
1036 Some(entry.reservation.child_session_id),
1037 child.short_id.as_deref(),
1038 "delegate run event confirmation failed",
1039 );
1040 let _ = self
1041 .resolve_delegate_failure_if_pending(
1042 entry.reservation.parent_session_id,
1043 entry.reservation.parent_run_id,
1044 entry.reservation.parent_tool_call_id,
1045 result,
1046 )
1047 .await;
1048 }
1049 let delegation_events = self.inner.delegation_events.clone();
1050 let _ = self
1051 .spawn_admission_blocking(move || {
1052 delegation_events.mark_finished(invocation_id, SessionStatus::Failed)
1053 })
1054 .await;
1055 let terminalized = self
1056 .terminalize_child_without_run(
1057 child.session_id,
1058 SessionStatus::Failed,
1059 "delegate child startup failed",
1060 )
1061 .await;
1062 if let Some(record) = self
1063 .inner
1064 .delegation
1065 .by_session
1066 .lock()
1067 .map_err(|_| EngineError::ActorStopped)?
1068 .get_mut(&child.session_id)
1069 {
1070 record.state = DelegationState::Finished(SessionStatus::Failed);
1071 }
1072 drop(admission_guard);
1073 if invocation.background {
1074 if terminalized.is_ok() {
1075 self.finish_background_or_retry(child.session_id, invocation_id)
1076 .await;
1077 } else {
1078 let _ = self.start_queued_delegation(root_session_id).await;
1079 }
1080 }
1081 return Err(error);
1082 }
1083 };
1084 if let Some(record) = self
1085 .inner
1086 .delegation
1087 .by_session
1088 .lock()
1089 .map_err(|_| EngineError::ActorStopped)?
1090 .get_mut(&child.session_id)
1091 {
1092 record.child_run_id = Some(child_run_id);
1093 record.state = DelegationState::Running;
1094 record.monitor_started = invocation.background;
1095 }
1096 let handle = DelegateHandle {
1097 invocation_id,
1098 child_session_id: child.session_id,
1099 child_run_id: Some(child_run_id),
1100 };
1101 if invocation.background
1102 && let Err(error) = self.spawn_background_monitor(handle, None)
1103 {
1104 self.release_recovered_monitor(child.session_id, invocation_id);
1105 let _ = self.cancel_run_durably(
1106 child_run_id,
1107 Some("background delegate monitor failed to start".into()),
1108 );
1109 drop(admission_guard);
1110 self.finish_background_or_retry(child.session_id, invocation_id)
1111 .await;
1112 return Err(error);
1113 }
1114 Ok(handle)
1115 }
1116
1117 pub fn await_delegate(&self, handle: DelegateHandle) -> DelegateAwait {
1120 let engine = self.clone();
1121 DelegateAwait {
1122 future: Box::pin(async move { engine.await_delegate_inner(handle).await }),
1123 engine: self.clone(),
1124 runtime: self.inner.runtime.clone(),
1125 handle,
1126 completed: false,
1127 }
1128 }
1129
1130 pub(super) async fn await_delegate_inner(
1131 &self,
1132 handle: DelegateHandle,
1133 ) -> Result<ToolResult, EngineError> {
1134 loop {
1135 let child = match self.inner.store.get(handle.child_session_id) {
1136 Ok(child) => child,
1137 Err(_) => {
1138 return Ok(delegate_failure_result(
1139 Some(handle.child_session_id),
1140 None,
1141 "child session is missing",
1142 ));
1143 }
1144 };
1145 let status = handle
1146 .child_run_id
1147 .and_then(|run_id| child.runs.get(&run_id).map(|run| run.status))
1148 .unwrap_or(child.status);
1149 match status {
1150 SessionStatus::Running | SessionStatus::Idle => {
1151 let active = {
1152 self.inner
1153 .sessions
1154 .active
1155 .lock()
1156 .unwrap_or_else(|poisoned| poisoned.into_inner())
1157 .get(&handle.child_run_id.expect("running delegate has a run"))
1158 .cloned()
1159 };
1160 if let Some(active) = active {
1161 tokio::select! {
1164 () = active.cancellation.cancelled() => {},
1165 () = tokio::time::sleep(std::time::Duration::from_millis(25)) => {},
1166 }
1167 } else {
1168 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
1169 }
1170 }
1171 SessionStatus::Completed
1172 | SessionStatus::Cancelled
1173 | SessionStatus::Failed
1174 | SessionStatus::Interrupted => {
1175 let delegation_events = self.inner.delegation_events.clone();
1176 self.spawn_admission_blocking(move || {
1177 delegation_events.mark_finished(handle.invocation_id, status)
1178 })
1179 .await?;
1180 let result = terminal_delegate_result(&child, handle.child_run_id, status);
1181 let mut records = self
1182 .inner
1183 .delegation
1184 .by_session
1185 .lock()
1186 .map_err(|_| EngineError::ActorStopped)?;
1187 if let Some(record) = records
1188 .get_mut(&handle.child_session_id)
1189 .filter(|record| record.invocation_id == handle.invocation_id)
1190 {
1191 record.state = DelegationState::Finished(status);
1192 }
1193 drop(records);
1194 self.clear_delegate_admissions(handle.invocation_id);
1195 return Ok(result);
1196 }
1197 }
1198 }
1199 }
1200
1201 pub async fn cancel_delegate(&self, handle: DelegateHandle) -> Result<ToolResult, EngineError> {
1204 let child = match self.inner.store.get(handle.child_session_id) {
1205 Ok(child) => child,
1206 Err(_) => {
1207 return Ok(delegate_failure_result(
1208 Some(handle.child_session_id),
1209 None,
1210 "child session is missing",
1211 ));
1212 }
1213 };
1214 if !matches!(child.status, SessionStatus::Running | SessionStatus::Idle) {
1215 return self.await_delegate(handle).await;
1216 }
1217 let Some(child_run_id) = handle.child_run_id else {
1218 return Ok(cancelled_delegate_result(
1219 handle.child_session_id,
1220 child.meta.short_id.as_deref(),
1221 None,
1222 ));
1223 };
1224 let _ = self.cancel_run(child_run_id).await;
1225 self.await_delegate(handle).await
1226 }
1227
1228 pub(super) async fn delegation_event_get(
1229 &self,
1230 invocation_id: InvocationId,
1231 ) -> Result<Option<delegation_events::DelegationEntry>, EngineError> {
1232 let delegation_events = self.inner.delegation_events.clone();
1233 self.spawn_admission_blocking(move || {
1234 Ok::<_, EngineError>(delegation_events.get(invocation_id))
1235 })
1236 .await
1237 }
1238
1239 pub(super) fn clear_delegate_admissions(&self, invocation_id: InvocationId) {
1240 self.inner
1241 .delegation
1242 .inflight
1243 .lock()
1244 .unwrap_or_else(|poisoned| poisoned.into_inner())
1245 .remove(&invocation_id);
1246 }
1247
1248 pub(super) async fn resolve_delegate_failure_if_pending(
1249 &self,
1250 session_id: SessionId,
1251 run_id: RunId,
1252 tool_call_id: ToolCallId,
1253 result: ToolResult,
1254 ) -> Result<bool, EngineError> {
1255 self.request(session_id, |reply| {
1256 SessionCommand::ResolveDelegateFailureIfPending {
1257 run: run_id,
1258 tool_call_id,
1259 result,
1260 reply,
1261 }
1262 })
1263 .await
1264 }
1265
1266 pub(super) fn resolve_delegate_failure_if_pending_direct(
1267 &self,
1268 session_id: SessionId,
1269 run_id: RunId,
1270 tool_call_id: ToolCallId,
1271 result: ToolResult,
1272 ) -> Result<bool, EngineError> {
1273 let pending = self
1274 .inner
1275 .store
1276 .get(session_id)?
1277 .runs
1278 .get(&run_id)
1279 .is_some_and(|run| {
1280 run.pending_calls
1281 .get(&tool_call_id)
1282 .is_some_and(|tool| tool == "delegate_subagent")
1283 });
1284 if !pending {
1285 return Ok(false);
1286 }
1287 self.append_direct(
1288 session_id,
1289 Some(run_id),
1290 super::event_origin("engine:tool-result"),
1291 Event::ToolCallTerminated {
1292 termination: ToolCallTermination {
1293 tool_call_id,
1294 owner: self.tool_call_owner(session_id, run_id, tool_call_id)?,
1295 outcome: ToolTerminationOutcome::Failed,
1296 result: Some(result),
1297 error: Some(SafeToolError {
1298 code: safe_code("delegate_failed"),
1299 message: safe_error("delegated run failed"),
1300 }),
1301 },
1302 },
1303 )?;
1304 Ok(true)
1305 }
1306
1307 pub(super) fn resolve_abandoned_delegate_failure_if_pending_direct(
1308 &self,
1309 invocation_id: InvocationId,
1310 generation: u64,
1311 session_id: SessionId,
1312 run_id: RunId,
1313 tool_call_id: ToolCallId,
1314 result: ToolResult,
1315 ) -> Result<bool, EngineError> {
1316 let mut admissions = self
1317 .inner
1318 .delegation
1319 .inflight
1320 .lock()
1321 .map_err(|_| EngineError::ActorStopped)?;
1322 let still_abandoned = admissions.get(&invocation_id).is_some_and(|entries| {
1323 entries
1324 .get(&generation)
1325 .is_some_and(|entry| entry.cancelled)
1326 && entries.values().all(|entry| entry.cancelled)
1327 });
1328 if !still_abandoned {
1329 return Ok(false);
1330 }
1331 let resolved = self.resolve_delegate_failure_if_pending_direct(
1332 session_id,
1333 run_id,
1334 tool_call_id,
1335 result,
1336 )?;
1337 admissions.remove(&invocation_id);
1338 Ok(resolved)
1339 }
1340
1341 pub(super) async fn ensure_delegate_run(
1342 &self,
1343 entry: &delegation_events::DelegationEntry,
1344 admission: Option<(InvocationId, u64)>,
1345 ) -> Result<RunId, EngineError> {
1346 if entry.terminal_status.is_some() {
1347 return Err(EngineError::ToolFailed(
1348 "delegate invocation was cancelled before startup".into(),
1349 ));
1350 }
1351 #[cfg(test)]
1352 if self
1353 .inner
1354 .test_hooks
1355 .delegate_start_failures
1356 .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
1357 (remaining > 0).then(|| remaining - 1)
1358 })
1359 .is_ok()
1360 {
1361 self.inner
1362 .test_hooks
1363 .delegate_start_failure_observed
1364 .notify_one();
1365 return Err(EngineError::ToolFailed(
1366 "injected delegate startup failure".into(),
1367 ));
1368 }
1369 if admission.is_some_and(|(invocation_id, generation)| {
1370 !self.admission_generation_live(invocation_id, generation)
1371 }) {
1372 return Err(EngineError::ToolFailed(
1373 "delegate cancelled before child start".into(),
1374 ));
1375 }
1376 let child = self.inner.store.get(entry.reservation.child_session_id)?;
1377 if let Some(staged_skill) = &entry.request.staged_skill {
1378 self.stage_child_skill_from_event(entry.reservation.child_session_id, staged_skill);
1379 }
1380 if let Some((invocation_id, generation)) = admission {
1381 self.mark_admission_starting(invocation_id, generation)?;
1382 }
1383 if let Some(run_id) = entry.child_run_id
1384 && child.runs.contains_key(&run_id)
1385 {
1386 if let Some((invocation_id, generation)) = admission {
1387 self.publish_admission_run(
1388 invocation_id,
1389 generation,
1390 entry.reservation.child_session_id,
1391 run_id,
1392 )?;
1393 if !self.admission_generation_live(invocation_id, generation) {
1394 self.sweep_abandoned_admission(invocation_id, generation)
1395 .await?;
1396 return Err(EngineError::ToolFailed(
1397 "delegate cancelled before child start".into(),
1398 ));
1399 }
1400 }
1401 return Ok(run_id);
1402 }
1403 let client_run_id = delegate_client_run_id(entry.reservation.invocation_id);
1404 let existing_run = child
1405 .runs
1406 .values()
1407 .find(|run| run.client_run_id == client_run_id)
1408 .map(|run| run.id);
1409 let run_id = match existing_run {
1410 Some(run_id) => run_id,
1411 None => match self
1412 .request(entry.reservation.child_session_id, |reply| {
1413 SessionCommand::Start {
1414 params: RunStartParams {
1415 reset_fallback: false,
1416 session_id: entry.reservation.child_session_id,
1417 client_run_id: client_run_id.clone(),
1418 selection: child.meta.creation_selection.clone(),
1419 input: render_delegate_input(&entry.request),
1420 },
1421 origin: super::event_origin("engine:delegation"),
1422 admission,
1423 reply,
1424 }
1425 })
1426 .await
1427 {
1428 Ok(started) => started.run_id,
1429 Err(error) => {
1430 if let Some((invocation_id, generation)) = admission {
1431 self.clear_admission_starting(invocation_id, generation);
1432 if !self.admission_generation_live(invocation_id, generation) {
1433 self.sweep_abandoned_admission(invocation_id, generation)
1434 .await?;
1435 }
1436 }
1437 return Err(error);
1438 }
1439 },
1440 };
1441 let cancelled = if let Some((invocation_id, generation)) = admission {
1442 self.publish_admission_run(
1445 invocation_id,
1446 generation,
1447 entry.reservation.child_session_id,
1448 run_id,
1449 )?;
1450 !self.admission_generation_live(invocation_id, generation)
1451 } else {
1452 false
1453 };
1454 #[cfg(test)]
1457 let confirmation_hook = self
1458 .inner
1459 .test_hooks
1460 .admission_confirmation_hook
1461 .lock()
1462 .expect("admission confirmation hook lock poisoned")
1463 .clone();
1464 #[cfg(test)]
1465 if let Some(hook) = confirmation_hook {
1466 let _ = hook.reached.send(());
1467 hook.release.wait().await;
1468 }
1469 let delegation_events = self.inner.delegation_events.clone();
1470 let invocation_id = entry.reservation.invocation_id;
1471 let confirmation = self
1472 .spawn_admission_blocking(move || {
1473 delegation_events.mark_run_started(invocation_id, run_id)
1474 })
1475 .await;
1476 if let Err(error) = confirmation {
1477 let _ = self.cancel_run_durably(
1480 run_id,
1481 Some("delegate run event confirmation failed".into()),
1482 );
1483 return Err(error);
1484 }
1485 if cancelled {
1486 if let Some((invocation_id, generation)) = admission {
1487 self.sweep_abandoned_admission(invocation_id, generation)
1488 .await?;
1489 }
1490 return Err(EngineError::ToolFailed(
1491 "delegate cancelled during child start".into(),
1492 ));
1493 }
1494 Ok(run_id)
1495 }
1496
1497 fn background_slot_unavailable(&self, root_session_id: SessionId) -> Result<bool, EngineError> {
1498 let Some(limit) = self.inner.config.runtime.delegation.max_concurrency else {
1499 return Ok(false);
1500 };
1501 let running = self
1502 .inner
1503 .delegation
1504 .by_session
1505 .lock()
1506 .map_err(|_| EngineError::ActorStopped)?
1507 .values()
1508 .filter(|record| {
1509 record.root_session_id == root_session_id
1510 && record.counts_slot
1511 && record.state == DelegationState::Running
1512 })
1513 .count() as u32;
1514 Ok(running >= limit)
1515 }
1516
1517 pub(super) fn interrupt_delegation_accounting(
1518 &self,
1519 parent_session_id: SessionId,
1520 interrupted_runs: &[RunId],
1521 ) -> Result<(), EngineError> {
1522 let mut records = self
1523 .inner
1524 .delegation
1525 .by_session
1526 .lock()
1527 .map_err(|_| EngineError::ActorStopped)?;
1528 for record in records.values_mut() {
1529 if record.parent_session_id == parent_session_id
1530 && interrupted_runs.contains(&record.parent_run_id)
1531 && record.state == DelegationState::Running
1532 {
1533 record.state = DelegationState::Finished(SessionStatus::Interrupted);
1534 }
1535 }
1536 Ok(())
1537 }
1538
1539 #[cfg(test)]
1540 pub(crate) fn running_background_delegations_for_test(
1541 &self,
1542 root_session_id: SessionId,
1543 ) -> usize {
1544 self.inner
1545 .delegation
1546 .by_session
1547 .lock()
1548 .expect("delegation registry")
1549 .values()
1550 .filter(|record| {
1551 record.root_session_id == root_session_id
1552 && record.counts_slot
1553 && record.state == DelegationState::Running
1554 })
1555 .count()
1556 }
1557
1558 fn background_queue_full(&self, root_session_id: SessionId) -> Result<bool, EngineError> {
1559 let Some(limit) = self.inner.config.runtime.delegation.max_concurrency else {
1560 return Ok(false);
1561 };
1562 let queued = self
1563 .inner
1564 .delegation
1565 .by_session
1566 .lock()
1567 .map_err(|_| EngineError::ActorStopped)?
1568 .values()
1569 .filter(|record| {
1570 record.root_session_id == root_session_id && record.state == DelegationState::Queued
1571 })
1572 .count() as u32;
1573 Ok(background_queue_limit_reached(limit, queued))
1574 }
1575
1576 fn spawn_background_monitor(
1582 &self,
1583 handle: DelegateHandle,
1584 recovery: Option<RecoveryMonitorTicket>,
1585 ) -> Result<(), EngineError> {
1586 let runtime = self
1587 .inner
1588 .runtime
1589 .clone()
1590 .or_else(|| tokio::runtime::Handle::try_current().ok())
1591 .ok_or(EngineError::ActorStopped)?;
1592 let engine = self.clone();
1593 let monitor = engine.clone();
1594 if !self.spawn_admission_task(&runtime, async move {
1595 let _recovery = recovery;
1596 let _ = monitor.await_delegate_inner(handle).await;
1597 loop {
1598 match monitor
1599 .finish_background_delegate(handle.child_session_id, handle.invocation_id)
1600 .await
1601 {
1602 Ok(()) => break,
1603 Err(error) => {
1604 if monitor
1605 .inner
1606 .delegation
1607 .admission_tasks_closing
1608 .load(Ordering::Acquire)
1609 {
1610 break;
1611 }
1612 eprintln!("background delegate completion retrying: {error}");
1613 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1614 }
1615 }
1616 }
1617 }) {
1618 return Err(EngineError::ActorStopped);
1619 }
1620 Ok(())
1621 }
1622
1623 async fn register_background_delegation_producer(
1624 &self,
1625 parent_session_id: SessionId,
1626 invocation_id: InvocationId,
1627 ) -> Result<BackgroundProducerGuard, EngineError> {
1628 let producer_id = self
1629 .register_producer(
1630 parent_session_id,
1631 delegation_producer_authority(invocation_id),
1632 )
1633 .await?;
1634 Ok(BackgroundProducerGuard {
1635 engine: self.clone(),
1636 runtime: self.inner.runtime.clone(),
1637 parent_session_id,
1638 invocation_id,
1639 producer_id,
1640 active: true,
1641 })
1642 }
1643
1644 fn spawn_background_monitor_gated(
1645 &self,
1646 handle: DelegateHandle,
1647 ) -> Result<tokio::sync::oneshot::Sender<()>, EngineError> {
1648 #[cfg(test)]
1649 if self
1650 .inner
1651 .test_hooks
1652 .resume_monitor_failures
1653 .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
1654 (remaining > 0).then(|| remaining - 1)
1655 })
1656 .is_ok()
1657 {
1658 return Err(EngineError::ActorStopped);
1659 }
1660 let runtime = self
1661 .inner
1662 .runtime
1663 .clone()
1664 .or_else(|| tokio::runtime::Handle::try_current().ok())
1665 .ok_or(EngineError::ActorStopped)?;
1666 let (release, admitted) = tokio::sync::oneshot::channel();
1667 let engine = self.clone();
1668 let monitor = engine.clone();
1669 if !self.spawn_admission_task(&runtime, async move {
1670 if admitted.await.is_err() {
1671 return;
1672 }
1673 let _ = monitor.await_delegate_inner(handle).await;
1674 loop {
1675 match monitor
1676 .finish_background_delegate(handle.child_session_id, handle.invocation_id)
1677 .await
1678 {
1679 Ok(()) => break,
1680 Err(error) => {
1681 if monitor
1682 .inner
1683 .delegation
1684 .admission_tasks_closing
1685 .load(Ordering::Acquire)
1686 {
1687 break;
1688 }
1689 eprintln!("background delegate completion retrying: {error}");
1690 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1691 }
1692 }
1693 }
1694 }) {
1695 return Err(EngineError::ActorStopped);
1696 }
1697 Ok(release)
1698 }
1699
1700 pub(super) async fn rollback_running_resume(
1701 &self,
1702 invocation_id: InvocationId,
1703 child_session_id: SessionId,
1704 child_run_id: RunId,
1705 admission_seq: u64,
1706 previous_record: DelegationRecord,
1707 status: SessionStatus,
1708 ) -> Result<(), EngineError> {
1709 let recalled = self
1710 .request(child_session_id, |reply| {
1711 SessionCommand::RecallDelegatedResume {
1712 run: child_run_id,
1713 admission_seq,
1714 reply,
1715 }
1716 })
1717 .await
1718 .unwrap_or(false);
1719 let delegation_events = self.inner.delegation_events.clone();
1720 self.spawn_admission_blocking(move || {
1721 delegation_events.mark_finished(invocation_id, status)
1722 })
1723 .await?;
1724 if let Ok(mut records) = self.inner.delegation.by_session.lock()
1725 && records
1726 .get(&child_session_id)
1727 .is_some_and(|record| record.invocation_id == invocation_id)
1728 {
1729 records.insert(child_session_id, previous_record);
1730 }
1731 if !recalled {
1732 let child = self.inner.store.get(child_session_id)?;
1733 if child
1734 .runs
1735 .get(&child_run_id)
1736 .is_some_and(|run| run.status == SessionStatus::Running)
1737 {
1738 let _ = self.cancel_run_durably(
1739 child_run_id,
1740 Some("abandoned resume prompt could not be recalled".into()),
1741 );
1742 }
1743 }
1744 Ok(())
1745 }
1746
1747 async fn finish_background_or_retry(
1748 &self,
1749 child_session_id: SessionId,
1750 invocation_id: InvocationId,
1751 ) {
1752 if self
1753 .finish_background_delegate(child_session_id, invocation_id)
1754 .await
1755 .is_err()
1756 {
1757 let _ = self.spawn_background_finish_retry(child_session_id, invocation_id);
1758 }
1759 }
1760
1761 fn spawn_background_finish_retry(
1762 &self,
1763 child_session_id: SessionId,
1764 invocation_id: InvocationId,
1765 ) -> Result<(), EngineError> {
1766 let runtime = self
1767 .inner
1768 .runtime
1769 .clone()
1770 .or_else(|| tokio::runtime::Handle::try_current().ok())
1771 .ok_or(EngineError::ActorStopped)?;
1772 let engine = self.clone();
1773 let retry = engine.clone();
1774 if !self.spawn_admission_task(&runtime, async move {
1775 loop {
1776 match retry
1777 .finish_background_delegate(child_session_id, invocation_id)
1778 .await
1779 {
1780 Ok(()) => break,
1781 Err(error) => {
1782 if retry
1783 .inner
1784 .delegation
1785 .admission_tasks_closing
1786 .load(Ordering::Acquire)
1787 {
1788 break;
1789 }
1790 eprintln!("background delegate completion retrying: {error}");
1791 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1792 }
1793 }
1794 }
1795 }) {
1796 return Err(EngineError::ActorStopped);
1797 }
1798 Ok(())
1799 }
1800
1801 async fn finish_background_delegate(
1802 &self,
1803 child_session_id: SessionId,
1804 invocation_id: InvocationId,
1805 ) -> Result<(), EngineError> {
1806 let child = self.inner.store.get(child_session_id)?;
1807 let Some(entry) = self.delegation_event_get(invocation_id).await? else {
1809 return Ok(());
1810 };
1811 if entry.reservation.child_session_id != child_session_id {
1812 return Err(EngineError::ToolFailed(
1813 "delegate reservation child does not match completion monitor".into(),
1814 ));
1815 }
1816 let parent_session_id = entry.reservation.parent_session_id;
1817 let parent = self.inner.store.get(parent_session_id)?;
1818 let root_session_id = match parent.meta.origin {
1819 SessionOrigin::Delegated {
1820 root_session_id, ..
1821 } => root_session_id,
1822 _ => parent_session_id,
1823 };
1824 let exact_status = if let Some(status) = entry.terminal_status {
1825 status
1826 } else if let Some(child_run_id) = entry.child_run_id {
1827 child
1828 .runs
1829 .get(&child_run_id)
1830 .map(|run| run.status)
1831 .ok_or_else(|| EngineError::ToolFailed("delegate child run is missing".into()))?
1832 } else {
1833 child.status
1834 };
1835 if !matches!(
1836 exact_status,
1837 SessionStatus::Completed
1838 | SessionStatus::Failed
1839 | SessionStatus::Interrupted
1840 | SessionStatus::Cancelled
1841 ) {
1842 return Err(EngineError::ToolFailed(
1843 "delegate child run is not terminal".into(),
1844 ));
1845 }
1846 let (owns_registry, mut producer_id) = {
1847 let mut records = self
1848 .inner
1849 .delegation
1850 .by_session
1851 .lock()
1852 .map_err(|_| EngineError::ActorStopped)?;
1853 if let Some(record) = records
1854 .get_mut(&child_session_id)
1855 .filter(|record| record.invocation_id == invocation_id)
1856 {
1857 record.state = DelegationState::Finished(exact_status);
1858 (true, record.producer_id)
1859 } else {
1860 (false, None)
1861 }
1862 };
1863 let parent_events = parent.log.event_snapshot();
1864 let mut already_logged = false;
1865 let mut already_accepted = false;
1866 for event in parent_events.iter() {
1867 match &event.payload {
1868 Event::DelegateFinishedV2 {
1869 invocation_id: logged_invocation,
1870 session_id,
1871 ..
1872 } if *logged_invocation == invocation_id && *session_id == child_session_id => {
1873 already_logged = true;
1874 }
1875 Event::ProducerMessageAccepted {
1876 producer_owner:
1877 ProducerOwner::Delegation {
1878 invocation_id: accepted_invocation,
1879 },
1880 ..
1881 } if *accepted_invocation == invocation_id => {
1882 already_accepted = true;
1883 }
1884 _ => {}
1885 }
1886 }
1887 let background = entry.request.background;
1888 let teaser = if entry.terminal_status.is_some() && entry.child_run_id.is_none() {
1889 DelegateTeaser {
1890 session_id: child_session_id,
1891 short_id: child
1892 .meta
1893 .short_id
1894 .clone()
1895 .unwrap_or_else(|| child_session_id.to_string()),
1896 status: SessionStatus::Cancelled,
1897 preview: String::new(),
1898 total_lines: 0,
1899 }
1900 } else {
1901 delegate_teaser(&child, child_session_id, exact_status, entry.child_run_id)
1902 };
1903 let mut transient_guard = None;
1906 if background && !already_accepted && !already_logged && producer_id.is_none() {
1907 let mut guard = self
1908 .register_background_delegation_producer(parent_session_id, invocation_id)
1909 .await?;
1910 let registered_id = guard.producer_id;
1911 producer_id = Some(registered_id);
1912 if owns_registry {
1913 let mut records = self
1914 .inner
1915 .delegation
1916 .by_session
1917 .lock()
1918 .map_err(|_| EngineError::ActorStopped)?;
1919 if let Some(record) = records
1920 .get_mut(&child_session_id)
1921 .filter(|record| record.invocation_id == invocation_id)
1922 {
1923 if record.producer_id.is_none() {
1924 record.producer_id = Some(registered_id);
1925 guard.handoff();
1926 }
1927 producer_id = record.producer_id;
1928 }
1929 }
1930 transient_guard = Some(guard);
1931 }
1932 if background && !already_logged {
1933 let send = self
1934 .commit_background_delegation_completion(
1935 entry.reservation.clone(),
1936 producer_id,
1937 teaser.clone(),
1938 )
1939 .await;
1940 let committed = if let Err(EngineError::Producer(_)) = send {
1941 let failed_id = producer_id;
1942 let mut guard = self
1943 .register_background_delegation_producer(parent_session_id, invocation_id)
1944 .await?;
1945 let registered_id = guard.producer_id;
1946 producer_id = Some(registered_id);
1947 if owns_registry {
1948 let mut records = self
1949 .inner
1950 .delegation
1951 .by_session
1952 .lock()
1953 .map_err(|_| EngineError::ActorStopped)?;
1954 if let Some(record) = records
1955 .get_mut(&child_session_id)
1956 .filter(|record| record.invocation_id == invocation_id)
1957 {
1958 if record.producer_id.is_none() || record.producer_id == failed_id {
1959 record.producer_id = Some(registered_id);
1960 guard.handoff();
1961 }
1962 producer_id = record.producer_id;
1963 }
1964 }
1965 transient_guard = Some(guard);
1966 self.commit_background_delegation_completion(
1967 entry.reservation.clone(),
1968 producer_id,
1969 teaser,
1970 )
1971 .await?
1972 } else {
1973 send?
1974 };
1975 if !committed {
1976 return Ok(());
1977 }
1978 already_accepted = true;
1979 already_logged = true;
1980 }
1981 if owns_registry
1982 && (already_accepted || already_logged)
1983 && let Some(record) = self
1984 .inner
1985 .delegation
1986 .by_session
1987 .lock()
1988 .map_err(|_| EngineError::ActorStopped)?
1989 .get_mut(&child_session_id)
1990 {
1991 record.notification_sent = true;
1992 }
1993 if let Some(id) = producer_id {
1994 match self
1995 .unregister_producer(
1996 parent_session_id,
1997 delegation_producer_authority(invocation_id),
1998 id,
1999 )
2000 .await
2001 {
2002 Ok(()) | Err(EngineError::Producer(_)) => {
2003 if let Some(guard) = transient_guard.as_mut() {
2004 guard.handoff();
2005 }
2006 if owns_registry
2007 && let Some(record) = self
2008 .inner
2009 .delegation
2010 .by_session
2011 .lock()
2012 .map_err(|_| EngineError::ActorStopped)?
2013 .get_mut(&child_session_id)
2014 {
2015 record.producer_id = None;
2016 }
2017 }
2018 Err(error) => return Err(error),
2019 }
2020 }
2021 let drain_result = if owns_registry {
2022 self.start_queued_delegation(root_session_id).await
2023 } else {
2024 Ok(())
2025 };
2026 let completion = drain_result;
2027 if completion.is_ok()
2028 && let Err(error) = self.evict_idle_subagents().await
2029 {
2030 eprintln!("subagent session completion eviction failed: {error}");
2031 }
2032 completion
2033 }
2034
2035 async fn commit_background_delegation_completion(
2036 &self,
2037 reservation: cookie_agent_protocol::DelegationReservation,
2038 producer_id: Option<ProducerId>,
2039 teaser: DelegateTeaser,
2040 ) -> Result<bool, EngineError> {
2041 self.request(reservation.parent_session_id, |reply| {
2042 SessionCommand::Producer(
2043 super::producers::ProducerCommand::CommitDelegationCompletion {
2044 reservation,
2045 producer_id,
2046 teaser,
2047 reply,
2048 },
2049 )
2050 })
2051 .await
2052 }
2053
2054 async fn terminalize_child_without_run(
2055 &self,
2056 child_session_id: SessionId,
2057 status: SessionStatus,
2058 reason: &str,
2059 ) -> Result<(), EngineError> {
2060 #[cfg(test)]
2061 if self
2062 .inner
2063 .test_hooks
2064 .delegate_terminal_append_failures
2065 .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
2066 (remaining > 0).then(|| remaining - 1)
2067 })
2068 .is_ok()
2069 {
2070 return Err(EngineError::ToolFailed(
2071 "injected delegate terminal append failure".into(),
2072 ));
2073 }
2074 let child = self.inner.store.get(child_session_id)?;
2075 if child.status == SessionStatus::Idle {
2076 self.append(
2077 child_session_id,
2078 None,
2079 super::event_origin("engine:delegation"),
2080 Event::DelegateChildTerminated {
2081 status,
2082 reason: Some(safe_error(reason)),
2083 },
2084 )
2085 .await?;
2086 }
2087 Ok(())
2088 }
2089
2090 async fn void_runless_pending_inputs(
2091 &self,
2092 child_session_id: SessionId,
2093 ) -> Result<(), EngineError> {
2094 let events = self.inner.store.get(child_session_id)?.log.event_snapshot();
2095 let boundary = events
2096 .iter()
2097 .rev()
2098 .find(|event| {
2099 matches!(
2100 event.payload,
2101 Event::RunCompleted { .. }
2102 | Event::RunFailed { .. }
2103 | Event::RunCancelled { .. }
2104 | Event::RunInterrupted { .. }
2105 )
2106 })
2107 .map_or(0, |event| event.seq);
2108 let mut pending = std::collections::VecDeque::new();
2109 for event in events
2110 .iter()
2111 .filter(|event| event.seq > boundary && event.run_id.is_none())
2112 {
2113 match &event.payload {
2114 Event::UserInputAdmitted { input } => pending.push_back(input.clone()),
2115 Event::UserInputRecalled { .. } => {
2116 pending.pop_back();
2117 }
2118 _ => {}
2119 }
2120 }
2121 for input in pending.into_iter().rev() {
2122 self.append(
2123 child_session_id,
2124 None,
2125 super::event_origin("engine:delegation"),
2126 Event::UserInputRecalled { input },
2127 )
2128 .await?;
2129 }
2130 Ok(())
2131 }
2132
2133 async fn start_queued_delegation(&self, root_session_id: SessionId) -> Result<(), EngineError> {
2134 let admission_guard = self.inner.delegation.admission.lock().await;
2135 if self.background_slot_unavailable(root_session_id)? {
2136 return Ok(());
2137 }
2138 let child_session_id = {
2139 let records = self
2140 .inner
2141 .delegation
2142 .by_session
2143 .lock()
2144 .map_err(|_| EngineError::ActorStopped)?;
2145 let queue = self
2146 .inner
2147 .delegation
2148 .queue
2149 .lock()
2150 .map_err(|_| EngineError::ActorStopped)?;
2151 let position = queue.iter().position(|session_id| {
2152 records.get(session_id).is_some_and(|record| {
2153 record.root_session_id == root_session_id
2154 && record.state == DelegationState::Queued
2155 })
2156 });
2157 position.and_then(|position| queue.get(position).copied())
2158 };
2159 let Some(child_session_id) = child_session_id else {
2160 return Ok(());
2161 };
2162 let invocation_id = self
2163 .inner
2164 .delegation
2165 .by_session
2166 .lock()
2167 .map_err(|_| EngineError::ActorStopped)?
2168 .get(&child_session_id)
2169 .map(|record| record.invocation_id)
2170 .ok_or_else(|| {
2171 EngineError::ToolFailed("queued subagent registry entry is missing".into())
2172 })?;
2173 let entry = self
2174 .delegation_event_get(invocation_id)
2175 .await?
2176 .ok_or_else(|| {
2177 EngineError::ToolFailed("queued subagent reservation is missing".into())
2178 })?;
2179 let child_run_id = self.ensure_delegate_run(&entry, None).await?;
2180 let removed = {
2181 let mut records = self
2182 .inner
2183 .delegation
2184 .by_session
2185 .lock()
2186 .map_err(|_| EngineError::ActorStopped)?;
2187 let record = records.get_mut(&child_session_id).ok_or_else(|| {
2188 EngineError::ToolFailed("queued subagent registry entry is missing".into())
2189 })?;
2190 record.child_run_id = Some(child_run_id);
2191 record.state = DelegationState::Running;
2192 record.monitor_started = true;
2193 let mut queue = self
2194 .inner
2195 .delegation
2196 .queue
2197 .lock()
2198 .map_err(|_| EngineError::ActorStopped)?;
2199 queue
2200 .iter()
2201 .position(|session_id| *session_id == child_session_id)
2202 .is_some_and(|position| queue.remove(position).is_some())
2203 };
2204 if !removed {
2205 let error =
2206 EngineError::ToolFailed("queued subagent disappeared during startup".into());
2207 let _ = self.cancel_run_durably(
2208 child_run_id,
2209 Some("queued subagent disappeared during startup".into()),
2210 );
2211 drop(admission_guard);
2212 let _ = self.spawn_background_finish_retry(child_session_id, invocation_id);
2213 return Err(error);
2214 }
2215 let handle = DelegateHandle {
2216 invocation_id,
2217 child_session_id,
2218 child_run_id: Some(child_run_id),
2219 };
2220 if let Err(error) = self.spawn_background_monitor(handle, None) {
2224 self.release_recovered_monitor(child_session_id, invocation_id);
2225 let _ = self.cancel_run_durably(
2226 child_run_id,
2227 Some("background delegate monitor failed to start".into()),
2228 );
2229 drop(admission_guard);
2230 let _ = self.spawn_background_finish_retry(child_session_id, invocation_id);
2231 return Err(error);
2232 }
2233 Ok(())
2234 }
2235
2236 pub fn subagent_agent_type(
2237 &self,
2238 caller_session_id: SessionId,
2239 child_session_id: SessionId,
2240 ) -> Result<cookie_agent_protocol::AgentId, EngineError> {
2241 let child = self.inner.store.get(child_session_id)?;
2242 match child.meta.origin {
2243 SessionOrigin::Delegated {
2244 parent_session_id, ..
2245 } if parent_session_id == caller_session_id => {
2246 Ok(child.meta.creation_selection.agent.clone())
2247 }
2248 _ => Err(EngineError::ToolFailed(
2249 "subagent session is not owned by the caller".into(),
2250 )),
2251 }
2252 }
2253
2254 async fn ensure_subagent_owned(
2255 &self,
2256 caller_session_id: SessionId,
2257 child_session_id: SessionId,
2258 ) -> Result<DelegateHandle, EngineError> {
2259 self.subagent_agent_type(caller_session_id, child_session_id)?;
2260 let child = self.inner.store.get(child_session_id)?;
2261 let SessionOrigin::Delegated {
2262 parent_run_id,
2263 parent_tool_call_id,
2264 invocation_id,
2265 ..
2266 } = child.meta.origin
2267 else {
2268 unreachable!("ownership check requires a delegated origin");
2269 };
2270 let registry_record = self
2271 .inner
2272 .delegation
2273 .by_session
2274 .lock()
2275 .map_err(|_| EngineError::ActorStopped)?
2276 .get(&child_session_id)
2277 .copied();
2278 if registry_record.is_some_and(|record| record.parent_session_id != caller_session_id) {
2279 return Err(EngineError::ToolFailed(
2280 "subagent registry ownership does not match its parent link".into(),
2281 ));
2282 }
2283 let (parent_run_id, parent_tool_call_id, invocation_id) = registry_record.map_or(
2284 (parent_run_id, parent_tool_call_id, invocation_id),
2285 |record| {
2286 (
2287 record.parent_run_id,
2288 record.parent_tool_call_id,
2289 record.invocation_id,
2290 )
2291 },
2292 );
2293 self.ensure_parent_link(
2294 caller_session_id,
2295 parent_run_id,
2296 parent_tool_call_id,
2297 child_session_id,
2298 )
2299 .await?;
2300 let child_run_id = registry_record
2301 .and_then(|record| record.child_run_id)
2302 .or_else(|| {
2303 self.inner
2304 .delegation_events
2305 .get(invocation_id)
2306 .and_then(|entry| entry.child_run_id)
2307 });
2308 Ok(DelegateHandle {
2309 invocation_id,
2310 child_session_id,
2311 child_run_id,
2312 })
2313 }
2314
2315 pub async fn get_subagent_result(
2320 &self,
2321 caller_session_id: SessionId,
2322 reference: impl std::fmt::Display,
2323 wait: bool,
2324 offset: u32,
2325 limit: u32,
2326 cancellation: CancellationToken,
2327 ) -> Result<ToolResult, EngineError> {
2328 let child_session_id = self.resolve_subagent_target(
2329 caller_session_id,
2330 reference,
2331 SubagentScope::DirectChildren,
2332 )?;
2333 self.ensure_subagent_owned(caller_session_id, child_session_id)
2334 .await?;
2335 loop {
2336 let child = self.inner.store.get(child_session_id)?;
2337 let running = matches!(child.status, SessionStatus::Running | SessionStatus::Idle)
2343 || super::producers::has_pending_producer_message(&child.log.event_snapshot())
2344 || self.producer_wake_in_flight(child_session_id);
2345 if !running {
2346 let text = last_assistant_message(&child);
2347 return Ok(paginated_subagent_result(
2348 session_status_name(child.status),
2349 &text,
2350 offset,
2351 limit,
2352 ));
2353 }
2354 if !wait {
2355 return Ok(paginated_subagent_result("running", "", offset, limit));
2356 }
2357 tokio::select! {
2358 () = cancellation.cancelled() => {
2359 return Err(EngineError::ToolFailed("subagent result wait cancelled".into()));
2360 }
2361 () = tokio::time::sleep(std::time::Duration::from_millis(25)) => {}
2362 }
2363 }
2364 }
2365
2366 pub async fn steer_subagent(
2367 &self,
2368 caller_session_id: SessionId,
2369 child_session_id: SessionId,
2370 message: String,
2371 ) -> Result<ToolResult, EngineError> {
2372 if message.trim().is_empty() {
2373 return Err(EngineError::ToolFailed(
2374 "subagent steer message must not be empty".into(),
2375 ));
2376 }
2377 let handle = self
2378 .ensure_subagent_owned(caller_session_id, child_session_id)
2379 .await?;
2380 let admission_guard = self.inner.delegation.admission.lock().await;
2381 let record = self
2382 .inner
2383 .delegation
2384 .by_session
2385 .lock()
2386 .map_err(|_| EngineError::ActorStopped)?
2387 .get(&child_session_id)
2388 .copied()
2389 .ok_or_else(|| EngineError::ToolFailed("subagent registry entry is missing".into()))?;
2390 let child = self.inner.store.get(child_session_id)?;
2391 match record.state {
2392 DelegationState::Queued => {
2393 self.append(
2394 child_session_id,
2395 None,
2396 super::event_origin("engine:delegation"),
2397 Event::UserInputAdmitted { input: message },
2398 )
2399 .await?;
2400 drop(admission_guard);
2401 Ok(steered_delegate_result(
2402 child_session_id,
2403 child.meta.short_id.as_deref(),
2404 "queued",
2405 ))
2406 }
2407 DelegationState::Starting | DelegationState::Running => {
2408 if matches!(
2409 child.status,
2410 SessionStatus::Completed
2411 | SessionStatus::Failed
2412 | SessionStatus::Interrupted
2413 | SessionStatus::Cancelled
2414 ) {
2415 return Err(EngineError::ToolFailed(format!(
2416 "subagent is terminal ({}) and cannot be steered",
2417 session_status_name(child.status)
2418 )));
2419 }
2420 let child_run_id =
2421 record.child_run_id.or(handle.child_run_id).ok_or_else(|| {
2422 EngineError::ToolFailed("subagent has not started a run".into())
2423 })?;
2424 drop(admission_guard);
2425 let result = self
2426 .request(child_session_id, |reply| SessionCommand::Steer {
2427 run: child_run_id,
2428 origin: super::event_origin("engine:delegation"),
2429 input: message,
2430 original_input: None,
2431 reply,
2432 })
2433 .await?;
2434 if !result.accepted {
2435 return Err(EngineError::ToolFailed(
2436 "subagent is no longer running".into(),
2437 ));
2438 }
2439 Ok(steered_delegate_result(
2440 child_session_id,
2441 child.meta.short_id.as_deref(),
2442 "running",
2443 ))
2444 }
2445 DelegationState::Finished(status) => Err(EngineError::ToolFailed(format!(
2446 "subagent is terminal ({}) and cannot be steered",
2447 session_status_name(status)
2448 ))),
2449 }
2450 }
2451
2452 pub async fn cancel_subagent(
2453 &self,
2454 caller_session_id: SessionId,
2455 reference: impl std::fmt::Display,
2456 reason: Option<String>,
2457 ) -> Result<ToolResult, EngineError> {
2458 let child_session_id = self.resolve_subagent_target(
2459 caller_session_id,
2460 reference,
2461 SubagentScope::DirectChildren,
2462 )?;
2463 let handle = self
2464 .ensure_subagent_owned(caller_session_id, child_session_id)
2465 .await?;
2466 let admission_guard = self.inner.delegation.admission.lock().await;
2467 let record = self
2468 .inner
2469 .delegation
2470 .by_session
2471 .lock()
2472 .map_err(|_| EngineError::ActorStopped)?
2473 .get(&child_session_id)
2474 .copied()
2475 .ok_or_else(|| EngineError::ToolFailed("subagent registry entry is missing".into()))?;
2476 if record.state == DelegationState::Queued {
2477 self.inner
2478 .delegation
2479 .by_session
2480 .lock()
2481 .map_err(|_| EngineError::ActorStopped)?
2482 .get(&child_session_id)
2483 .filter(|record| record.state == DelegationState::Queued)
2484 .map(|_| ())
2485 .ok_or_else(|| {
2486 EngineError::ToolFailed("subagent is not queued for cancellation".into())
2487 })?;
2488 let cancellation_reason = reason
2489 .as_deref()
2490 .unwrap_or("queued subagent cancelled before startup");
2491 let delegation_events = self.inner.delegation_events.clone();
2492 self.spawn_admission_blocking(move || {
2493 delegation_events.mark_finished(record.invocation_id, SessionStatus::Cancelled)
2494 })
2495 .await?;
2496 self.void_runless_pending_inputs(child_session_id).await?;
2497 self.terminalize_child_without_run(
2498 child_session_id,
2499 SessionStatus::Cancelled,
2500 cancellation_reason,
2501 )
2502 .await?;
2503 self.inner
2504 .delegation
2505 .queue
2506 .lock()
2507 .map_err(|_| EngineError::ActorStopped)?
2508 .retain(|session_id| *session_id != child_session_id);
2509 if let Some(record) = self
2510 .inner
2511 .delegation
2512 .by_session
2513 .lock()
2514 .map_err(|_| EngineError::ActorStopped)?
2515 .get_mut(&child_session_id)
2516 {
2517 record.state = DelegationState::Finished(SessionStatus::Cancelled);
2518 }
2519 drop(admission_guard);
2520 self.finish_background_or_retry(child_session_id, record.invocation_id)
2521 .await;
2522 let short_id = self.session_short_id(child_session_id);
2523 return Ok(cancelled_delegate_result(
2524 child_session_id,
2525 short_id.as_deref(),
2526 None,
2527 ));
2528 }
2529 let child_run_id = record
2530 .child_run_id
2531 .or(handle.child_run_id)
2532 .or_else(|| {
2533 self.inner
2534 .delegation_events
2535 .get(handle.invocation_id)
2536 .and_then(|entry| entry.child_run_id)
2537 })
2538 .ok_or_else(|| EngineError::ToolFailed("subagent has not started a run".into()))?;
2539 drop(admission_guard);
2540 let child = self.inner.store.get(child_session_id)?;
2541 if matches!(child.status, SessionStatus::Running | SessionStatus::Idle) {
2542 self.cancel_run_durably(child_run_id, reason)?;
2543 }
2544 loop {
2545 let child = self.inner.store.get(child_session_id)?;
2546 if !matches!(child.status, SessionStatus::Running | SessionStatus::Idle) {
2547 return Ok(cancelled_delegate_result(
2548 child_session_id,
2549 child.meta.short_id.as_deref(),
2550 Some(delegate_final_text(&child, Some(child_run_id)).to_owned()),
2551 ));
2552 }
2553 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
2554 }
2555 }
2556
2557 #[cfg(test)]
2558 pub(crate) fn delegation_queue_contains(
2559 &self,
2560 child_session_id: SessionId,
2561 ) -> Result<bool, EngineError> {
2562 Ok(self
2563 .inner
2564 .delegation
2565 .queue
2566 .lock()
2567 .map_err(|_| EngineError::ActorStopped)?
2568 .contains(&child_session_id))
2569 }
2570
2571 #[cfg(test)]
2572 pub(crate) async fn wait_for_delegation_reconciliation_for_test(&self) {
2573 while self
2574 .inner
2575 .delegation
2576 .reconciliation_running
2577 .load(Ordering::Acquire)
2578 || self
2579 .inner
2580 .delegation
2581 .reconciliation_requested
2582 .load(Ordering::Acquire)
2583 {
2584 tokio::task::yield_now().await;
2585 }
2586 }
2587
2588 #[cfg(test)]
2589 pub(crate) fn delegation_registry_snapshot(
2590 &self,
2591 child_session_id: SessionId,
2592 ) -> Result<(InvocationId, Option<SessionStatus>, bool), EngineError> {
2593 self.inner
2594 .delegation
2595 .by_session
2596 .lock()
2597 .map_err(|_| EngineError::ActorStopped)?
2598 .get(&child_session_id)
2599 .map(|record| {
2600 (
2601 record.invocation_id,
2602 match record.state {
2603 DelegationState::Finished(status) => Some(status),
2604 _ => None,
2605 },
2606 record.counts_slot,
2607 )
2608 })
2609 .ok_or_else(|| EngineError::ToolFailed("subagent registry entry is missing".into()))
2610 }
2611
2612 #[cfg(test)]
2613 pub(crate) fn install_resume_admission_hook(
2614 &self,
2615 ) -> (tokio::sync::oneshot::Receiver<()>, Arc<tokio::sync::Notify>) {
2616 let (reached, receiver) = tokio::sync::oneshot::channel();
2617 let release = Arc::new(tokio::sync::Notify::new());
2618 *self
2619 .inner
2620 .test_hooks
2621 .resume_admission_hook
2622 .lock()
2623 .expect("resume admission hook lock poisoned") =
2624 Some(Arc::new(super::ResumeAdmissionHook {
2625 reached: std::sync::Mutex::new(Some(reached)),
2626 release: Arc::clone(&release),
2627 }));
2628 (receiver, release)
2629 }
2630
2631 #[cfg(test)]
2632 pub(crate) fn install_skill_fork_reservation_hook(
2633 &self,
2634 ) -> (oneshot::Receiver<()>, Arc<tokio::sync::Notify>) {
2635 let (reached, receiver) = oneshot::channel();
2636 let release = Arc::new(tokio::sync::Notify::new());
2637 *self
2638 .inner
2639 .test_hooks
2640 .skill_fork_reservation_hook
2641 .lock()
2642 .expect("skill fork reservation hook lock poisoned") =
2643 Some(Arc::new(super::PagingRaceHook {
2644 reached: std::sync::Mutex::new(Some(reached)),
2645 release: Arc::clone(&release),
2646 }));
2647 (receiver, release)
2648 }
2649
2650 #[cfg(test)]
2651 pub(crate) fn install_delegation_reservation_hook(
2652 &self,
2653 ) -> (oneshot::Receiver<()>, Arc<tokio::sync::Notify>) {
2654 let (reached, receiver) = oneshot::channel();
2655 let release = Arc::new(tokio::sync::Notify::new());
2656 *self
2657 .inner
2658 .test_hooks
2659 .delegation_reservation_hook
2660 .lock()
2661 .expect("delegation reservation hook lock poisoned") =
2662 Some(Arc::new(super::PagingRaceHook {
2663 reached: std::sync::Mutex::new(Some(reached)),
2664 release: Arc::clone(&release),
2665 }));
2666 (receiver, release)
2667 }
2668
2669 #[cfg(test)]
2670 pub(crate) fn install_resume_attachment_hook(
2671 &self,
2672 ) -> (tokio::sync::oneshot::Receiver<()>, Arc<tokio::sync::Notify>) {
2673 self.install_resume_test_hook(&self.inner.test_hooks.resume_attachment_hook)
2674 }
2675
2676 #[cfg(test)]
2677 pub(crate) fn install_resume_rollback_hook(
2678 &self,
2679 ) -> (tokio::sync::oneshot::Receiver<()>, Arc<tokio::sync::Notify>) {
2680 self.install_resume_test_hook(&self.inner.test_hooks.resume_rollback_hook)
2681 }
2682
2683 #[cfg(test)]
2684 fn install_resume_test_hook(
2685 &self,
2686 slot: &std::sync::Mutex<Option<Arc<super::ResumeAdmissionHook>>>,
2687 ) -> (tokio::sync::oneshot::Receiver<()>, Arc<tokio::sync::Notify>) {
2688 let (reached, receiver) = tokio::sync::oneshot::channel();
2689 let release = Arc::new(tokio::sync::Notify::new());
2690 *slot.lock().expect("resume test hook lock poisoned") =
2691 Some(Arc::new(super::ResumeAdmissionHook {
2692 reached: std::sync::Mutex::new(Some(reached)),
2693 release: Arc::clone(&release),
2694 }));
2695 (receiver, release)
2696 }
2697
2698 #[cfg(test)]
2699 async fn wait_for_resume_test_hook(
2700 &self,
2701 slot: &std::sync::Mutex<Option<Arc<super::ResumeAdmissionHook>>>,
2702 ) {
2703 let hook = slot.lock().expect("resume test hook lock poisoned").clone();
2704 if let Some(hook) = hook {
2705 if let Some(reached) = hook
2706 .reached
2707 .lock()
2708 .expect("resume test hook reached lock poisoned")
2709 .take()
2710 {
2711 let _ = reached.send(());
2712 }
2713 hook.release.notified().await;
2714 }
2715 }
2716
2717 #[cfg(test)]
2718 pub(crate) fn set_delegation_slot_ownership(
2719 &self,
2720 child_session_id: SessionId,
2721 background: bool,
2722 counts_slot: bool,
2723 ) -> Result<(), EngineError> {
2724 let mut records = self
2725 .inner
2726 .delegation
2727 .by_session
2728 .lock()
2729 .map_err(|_| EngineError::ActorStopped)?;
2730 let record = records
2731 .get_mut(&child_session_id)
2732 .ok_or_else(|| EngineError::ToolFailed("subagent registry entry is missing".into()))?;
2733 record.background = background;
2734 record.counts_slot = counts_slot;
2735 Ok(())
2736 }
2737
2738 #[cfg(test)]
2742 pub(crate) async fn plant_foreground_delegation_producer_for_test(
2743 &self,
2744 child_session_id: SessionId,
2745 ) -> Result<ProducerId, EngineError> {
2746 let (parent_session_id, invocation_id) = {
2747 let records = self
2748 .inner
2749 .delegation
2750 .by_session
2751 .lock()
2752 .map_err(|_| EngineError::ActorStopped)?;
2753 let record = records.get(&child_session_id).ok_or_else(|| {
2754 EngineError::ToolFailed("subagent registry entry is missing".into())
2755 })?;
2756 (record.parent_session_id, record.invocation_id)
2757 };
2758 let mut guard = self
2759 .register_background_delegation_producer(parent_session_id, invocation_id)
2760 .await?;
2761 let producer_id = guard.producer_id;
2762 {
2763 let mut records = self
2764 .inner
2765 .delegation
2766 .by_session
2767 .lock()
2768 .map_err(|_| EngineError::ActorStopped)?;
2769 let record = records
2770 .get_mut(&child_session_id)
2771 .filter(|record| record.invocation_id == invocation_id)
2772 .ok_or_else(|| {
2773 EngineError::ToolFailed("subagent registry entry is missing".into())
2774 })?;
2775 record.background = false;
2776 record.producer_id = Some(producer_id);
2777 }
2778 guard.handoff();
2779 Ok(producer_id)
2780 }
2781
2782 #[cfg(test)]
2783 pub(crate) fn rebuild_delegation_registry_for_test(&self) -> Result<(), EngineError> {
2784 self.rebuild_delegation_registry(&self.inner.delegation_events.entries(), true)
2785 }
2786
2787 #[cfg(test)]
2788 pub(crate) fn delegation_producer_ownership_for_test(
2789 &self,
2790 child_session_id: SessionId,
2791 ) -> Result<(bool, Option<ProducerId>), EngineError> {
2792 self.inner
2793 .delegation
2794 .by_session
2795 .lock()
2796 .map_err(|_| EngineError::ActorStopped)?
2797 .get(&child_session_id)
2798 .map(|record| (record.background, record.producer_id))
2799 .ok_or_else(|| EngineError::ToolFailed("subagent registry entry is missing".into()))
2800 }
2801
2802 pub(super) fn rebuild_delegation_registry(
2803 &self,
2804 entries: &[delegation_events::DelegationEntry],
2805 recover: bool,
2806 ) -> Result<(), EngineError> {
2807 let mut latest = std::collections::HashMap::new();
2808 for entry in entries {
2809 latest.insert(entry.reservation.child_session_id, entry);
2810 }
2811 let mut stale_producers = {
2812 let mut records = self
2813 .inner
2814 .delegation
2815 .by_session
2816 .lock()
2817 .map_err(|_| EngineError::ActorStopped)?;
2818 let stale = records
2819 .iter()
2820 .filter(|(child_session_id, record)| {
2821 latest
2822 .get(child_session_id)
2823 .is_none_or(|entry| entry.reservation.invocation_id != record.invocation_id)
2824 })
2825 .filter_map(|(_, record)| {
2826 record.producer_id.map(|producer_id| {
2827 (record.parent_session_id, record.invocation_id, producer_id)
2828 })
2829 })
2830 .collect::<Vec<_>>();
2831 records.retain(|child_session_id, record| {
2832 latest
2833 .get(child_session_id)
2834 .is_some_and(|entry| entry.reservation.invocation_id == record.invocation_id)
2835 });
2836 stale
2837 };
2838 self.inner
2839 .delegation
2840 .queue
2841 .lock()
2842 .map_err(|_| EngineError::ActorStopped)?
2843 .retain(|child_session_id| latest.contains_key(child_session_id));
2844 for entry in latest.into_values() {
2845 let child_id = entry.reservation.child_session_id;
2846 if !self.inner.store.session_exists(child_id) {
2850 continue;
2851 }
2852 let Some(facts) = self
2866 .inner
2867 .store
2868 .parent_run_facts(entry.reservation.parent_session_id)?
2869 else {
2870 continue;
2871 };
2872 let root_session_id = facts.root_session_id;
2873 let queued_event = facts.queued.iter().any(|(run_id, session)| {
2874 *run_id == entry.reservation.parent_run_id && *session == child_id
2875 });
2876 let notification_sent = facts.notified.iter().any(|(invocation_id, session)| {
2877 *invocation_id == entry.reservation.invocation_id && *session == child_id
2878 }) || facts
2879 .producer_accepted
2880 .contains(&entry.reservation.invocation_id);
2881 let background = entry.request.background;
2882 let exact_run_status = entry
2885 .child_run_id
2886 .and_then(|run_id| self.inner.store.terminal_run_status(child_id, run_id));
2887 let parent_interrupted = facts
2888 .interrupted_runs
2889 .contains(&entry.reservation.parent_run_id);
2890 let state = if let Some(status) = entry.terminal_status {
2891 DelegationState::Finished(status)
2892 } else if entry.child_run_id.is_none() && queued_event {
2893 DelegationState::Queued
2894 } else if parent_interrupted {
2895 DelegationState::Finished(SessionStatus::Interrupted)
2896 } else if exact_run_status.is_some_and(|status| {
2897 matches!(
2898 status,
2899 SessionStatus::Completed
2900 | SessionStatus::Failed
2901 | SessionStatus::Interrupted
2902 | SessionStatus::Cancelled
2903 )
2904 }) {
2905 DelegationState::Finished(exact_run_status.expect("checked terminal status"))
2906 } else {
2907 DelegationState::Running
2908 };
2909 let counts_slot = background && !facts.delegated;
2910 let (mut producer_id, mut monitor_started) = self
2911 .inner
2912 .delegation
2913 .by_session
2914 .lock()
2915 .map_err(|_| EngineError::ActorStopped)?
2916 .get(&child_id)
2917 .filter(|record| record.invocation_id == entry.reservation.invocation_id)
2918 .map_or((None, false), |record| {
2919 (record.producer_id, record.monitor_started)
2920 });
2921 let mut stale_foreground_producer = None;
2926 if !background && let Some(stale) = producer_id {
2927 stale_foreground_producer = Some(stale);
2928 producer_id = None;
2929 monitor_started = false;
2930 }
2931 let parent_session_id = entry.reservation.parent_session_id;
2932 let invocation_id = entry.reservation.invocation_id;
2933 self.inner
2934 .delegation
2935 .by_session
2936 .lock()
2937 .map_err(|_| EngineError::ActorStopped)?
2938 .insert(
2939 child_id,
2940 DelegationRecord {
2941 parent_session_id,
2942 parent_run_id: entry.reservation.parent_run_id,
2943 parent_tool_call_id: entry.reservation.parent_tool_call_id,
2944 invocation_id,
2945 root_session_id,
2946 child_run_id: entry.child_run_id,
2947 state,
2948 background,
2949 counts_slot,
2950 notification_sent,
2951 producer_id,
2952 monitor_started,
2953 },
2954 );
2955 if let Some(stale) = stale_foreground_producer {
2956 stale_producers.push((parent_session_id, invocation_id, stale));
2957 }
2958 if state == DelegationState::Queued {
2959 let mut queue = self
2960 .inner
2961 .delegation
2962 .queue
2963 .lock()
2964 .map_err(|_| EngineError::ActorStopped)?;
2965 if !queue.contains(&child_id) {
2966 queue.push_back(child_id);
2967 }
2968 }
2969 }
2970 if recover
2971 && let Some(runtime) = self
2972 .inner
2973 .runtime
2974 .clone()
2975 .or_else(|| tokio::runtime::Handle::try_current().ok())
2976 {
2977 self.schedule_delegation_reconciliation(&runtime, stale_producers)?;
2978 }
2979 Ok(())
2980 }
2981
2982 fn schedule_delegation_reconciliation(
2983 &self,
2984 runtime: &tokio::runtime::Handle,
2985 stale_producers: Vec<(SessionId, InvocationId, ProducerId)>,
2986 ) -> Result<(), EngineError> {
2987 self.inner
2988 .delegation
2989 .recovery_stale_producers
2990 .lock()
2991 .map_err(|_| EngineError::ActorStopped)?
2992 .extend(stale_producers);
2993 self.inner
2994 .delegation
2995 .reconciliation_requested
2996 .store(true, Ordering::Release);
2997 if self
2998 .inner
2999 .delegation
3000 .reconciliation_running
3001 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
3002 .is_err()
3003 {
3004 return Ok(());
3005 }
3006 let engine = self.clone();
3007 let reconcile = engine.clone();
3008 if !self.spawn_admission_task(runtime, async move {
3009 reconcile.run_delegation_reconciliation().await;
3010 }) {
3011 self.inner
3012 .delegation
3013 .reconciliation_running
3014 .store(false, Ordering::Release);
3015 self.inner.delegation.recovery_settled.notify_waiters();
3016 return Err(EngineError::ActorStopped);
3017 }
3018 Ok(())
3019 }
3020
3021 async fn run_delegation_reconciliation(&self) {
3022 loop {
3023 self.inner
3024 .delegation
3025 .reconciliation_requested
3026 .store(false, Ordering::Release);
3027 let stale_producers = std::mem::take(
3028 &mut *self
3029 .inner
3030 .delegation
3031 .recovery_stale_producers
3032 .lock()
3033 .unwrap_or_else(|poisoned| poisoned.into_inner()),
3034 );
3035 if let Err(error) = self.reconcile_delegation_producers(stale_producers).await {
3036 eprintln!("delegation producer recovery deferred: {error}");
3037 }
3038 if self
3039 .inner
3040 .delegation
3041 .reconciliation_requested
3042 .load(Ordering::Acquire)
3043 {
3044 continue;
3045 }
3046 self.inner
3047 .delegation
3048 .reconciliation_running
3049 .store(false, Ordering::Release);
3050 if !self
3051 .inner
3052 .delegation
3053 .reconciliation_requested
3054 .load(Ordering::Acquire)
3055 || self
3056 .inner
3057 .delegation
3058 .reconciliation_running
3059 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
3060 .is_err()
3061 {
3062 self.inner.delegation.recovery_settled.notify_waiters();
3065 break;
3066 }
3067 }
3068 }
3069
3070 async fn reconcile_delegation_producers(
3073 &self,
3074 stale_producers: Vec<(SessionId, InvocationId, ProducerId)>,
3075 ) -> Result<(), EngineError> {
3076 for (parent_session_id, invocation_id, producer_id) in stale_producers {
3077 if !self.inner.store.is_owned(parent_session_id) {
3078 continue;
3079 }
3080 match self
3081 .unregister_producer(
3082 parent_session_id,
3083 delegation_producer_authority(invocation_id),
3084 producer_id,
3085 )
3086 .await
3087 {
3088 Ok(()) | Err(EngineError::Producer(_)) => {}
3089 Err(error) => return Err(error),
3090 }
3091 }
3092 let records = self
3093 .inner
3094 .delegation
3095 .by_session
3096 .lock()
3097 .map_err(|_| EngineError::ActorStopped)?
3098 .iter()
3099 .map(|(session_id, record)| (*session_id, *record))
3100 .collect::<Vec<_>>();
3101
3102 for (child_session_id, record) in &records {
3103 if !record.background
3104 || record.notification_sent
3105 || record.producer_id.is_some()
3106 || !self.inner.store.is_owned(record.parent_session_id)
3107 {
3108 continue;
3109 }
3110 let mut guard = self
3111 .register_background_delegation_producer(
3112 record.parent_session_id,
3113 record.invocation_id,
3114 )
3115 .await?;
3116 let mut current = self
3117 .inner
3118 .delegation
3119 .by_session
3120 .lock()
3121 .map_err(|_| EngineError::ActorStopped)?;
3122 if let Some(current_record) = current.get_mut(child_session_id).filter(|current| {
3123 current.invocation_id == record.invocation_id && current.producer_id.is_none()
3124 }) {
3125 current_record.producer_id = Some(guard.producer_id);
3126 guard.handoff();
3127 }
3128 }
3129
3130 let records = self
3131 .inner
3132 .delegation
3133 .by_session
3134 .lock()
3135 .map_err(|_| EngineError::ActorStopped)?
3136 .iter()
3137 .map(|(session_id, record)| (*session_id, *record))
3138 .collect::<Vec<_>>();
3139 let mut queued_roots = Vec::new();
3140 let mut finished = Vec::new();
3141 for (child_session_id, record) in records {
3142 if !record.background || !self.inner.store.is_owned(record.parent_session_id) {
3143 continue;
3144 }
3145 match record.state {
3146 DelegationState::Finished(_) => {
3147 finished.push((child_session_id, record.invocation_id));
3148 }
3149 DelegationState::Queued => queued_roots.push(record.root_session_id),
3150 DelegationState::Running if record.child_run_id.is_some() => {
3151 let Some(ticket) =
3152 self.claim_recovered_monitor(child_session_id, record.invocation_id)?
3153 else {
3154 continue;
3155 };
3156 self.terminalize_abandoned_delegate_run(
3157 child_session_id,
3158 record.child_run_id.expect("checked child run"),
3159 )
3160 .await;
3161 if let Err(error) = self.spawn_background_monitor(
3162 DelegateHandle {
3163 invocation_id: record.invocation_id,
3164 child_session_id,
3165 child_run_id: record.child_run_id,
3166 },
3167 Some(ticket),
3168 ) {
3169 self.release_recovered_monitor(child_session_id, record.invocation_id);
3170 return Err(error);
3171 }
3172 }
3173 DelegationState::Running | DelegationState::Starting => {
3174 let Some(ticket) =
3175 self.claim_recovered_monitor(child_session_id, record.invocation_id)?
3176 else {
3177 continue;
3178 };
3179 let entry = self
3180 .delegation_event_get(record.invocation_id)
3181 .await
3182 .inspect_err(|_| {
3183 self.release_recovered_monitor(child_session_id, record.invocation_id);
3184 })?
3185 .ok_or_else(|| {
3186 self.release_recovered_monitor(child_session_id, record.invocation_id);
3187 EngineError::ToolFailed(
3188 "recovered subagent reservation is missing".into(),
3189 )
3190 })?;
3191 let child_run_id =
3192 self.ensure_delegate_run(&entry, None)
3193 .await
3194 .inspect_err(|_| {
3195 self.release_recovered_monitor(
3196 child_session_id,
3197 record.invocation_id,
3198 );
3199 })?;
3200 if let Some(current) = self
3201 .inner
3202 .delegation
3203 .by_session
3204 .lock()
3205 .map_err(|_| EngineError::ActorStopped)?
3206 .get_mut(&child_session_id)
3207 .filter(|current| current.invocation_id == record.invocation_id)
3208 {
3209 current.child_run_id = Some(child_run_id);
3210 current.state = DelegationState::Running;
3211 }
3212 if let Err(error) = self.spawn_background_monitor(
3213 DelegateHandle {
3214 invocation_id: record.invocation_id,
3215 child_session_id,
3216 child_run_id: Some(child_run_id),
3217 },
3218 Some(ticket),
3219 ) {
3220 self.release_recovered_monitor(child_session_id, record.invocation_id);
3221 return Err(error);
3222 }
3223 }
3224 }
3225 }
3226 for (child_session_id, invocation_id) in finished {
3227 self.finish_background_or_retry(child_session_id, invocation_id)
3228 .await;
3229 }
3230 queued_roots.sort_unstable();
3231 queued_roots.dedup();
3232 for root_session_id in queued_roots {
3233 self.start_queued_delegation(root_session_id).await?;
3234 }
3235 Ok(())
3236 }
3237
3238 async fn terminalize_abandoned_delegate_run(
3251 &self,
3252 child_session_id: SessionId,
3253 child_run_id: RunId,
3254 ) {
3255 let live = self
3256 .inner
3257 .sessions
3258 .active
3259 .lock()
3260 .unwrap_or_else(|poisoned| poisoned.into_inner())
3261 .contains_key(&child_run_id);
3262 if live {
3263 return;
3264 }
3265 let _residency = self.inner.sessions.residency_mutation.lock().await;
3266 if let Err(error) = self.ensure_session_owned(child_session_id) {
3267 eprintln!("session {child_session_id} delegate recovery deferred: {error}");
3268 }
3269 }
3270
3271 fn claim_recovered_monitor(
3277 &self,
3278 child_session_id: SessionId,
3279 invocation_id: InvocationId,
3280 ) -> Result<Option<RecoveryMonitorTicket>, EngineError> {
3281 let mut records = self
3282 .inner
3283 .delegation
3284 .by_session
3285 .lock()
3286 .map_err(|_| EngineError::ActorStopped)?;
3287 let Some(record) = records
3288 .get_mut(&child_session_id)
3289 .filter(|record| record.invocation_id == invocation_id)
3290 else {
3291 return Ok(None);
3292 };
3293 if record.monitor_started
3294 || !matches!(
3295 record.state,
3296 DelegationState::Running | DelegationState::Starting
3297 )
3298 {
3299 return Ok(None);
3300 }
3301 record.monitor_started = true;
3302 let root_session_id = record.root_session_id;
3303 drop(records);
3304 *self
3305 .inner
3306 .delegation
3307 .recovery_pending
3308 .lock()
3309 .unwrap_or_else(|poisoned| poisoned.into_inner())
3310 .entry(root_session_id)
3311 .or_insert(0) += 1;
3312 Ok(Some(RecoveryMonitorTicket {
3313 engine: self.clone(),
3314 root_session_id,
3315 }))
3316 }
3317
3318 fn release_recovery_claim(&self, root_session_id: SessionId) {
3319 {
3320 let mut pending = self
3321 .inner
3322 .delegation
3323 .recovery_pending
3324 .lock()
3325 .unwrap_or_else(|poisoned| poisoned.into_inner());
3326 if let Some(count) = pending.get_mut(&root_session_id) {
3327 *count = count.saturating_sub(1);
3328 if *count == 0 {
3329 pending.remove(&root_session_id);
3330 }
3331 }
3332 }
3333 self.inner.delegation.recovery_settled.notify_waiters();
3334 }
3335
3336 fn delegation_recovery_settled(&self, root_session_id: SessionId) -> bool {
3337 if self
3338 .inner
3339 .delegation
3340 .reconciliation_running
3341 .load(Ordering::Acquire)
3342 || self
3343 .inner
3344 .delegation
3345 .reconciliation_requested
3346 .load(Ordering::Acquire)
3347 {
3348 return false;
3349 }
3350 !self
3351 .inner
3352 .delegation
3353 .recovery_pending
3354 .lock()
3355 .unwrap_or_else(|poisoned| poisoned.into_inner())
3356 .contains_key(&root_session_id)
3357 }
3358
3359 pub(crate) async fn await_delegation_recovery(
3368 &self,
3369 root_session_id: SessionId,
3370 bound: std::time::Duration,
3371 ) -> bool {
3372 let settle = async {
3373 while !self.delegation_recovery_settled(root_session_id) {
3374 let _ = tokio::time::timeout(
3378 std::time::Duration::from_millis(10),
3379 self.inner.delegation.recovery_settled.notified(),
3380 )
3381 .await;
3382 }
3383 };
3384 tokio::time::timeout(bound, settle).await.is_ok()
3385 }
3386
3387 fn release_recovered_monitor(&self, child_session_id: SessionId, invocation_id: InvocationId) {
3388 if let Ok(mut records) = self.inner.delegation.by_session.lock()
3389 && let Some(record) = records
3390 .get_mut(&child_session_id)
3391 .filter(|record| record.invocation_id == invocation_id)
3392 {
3393 record.monitor_started = false;
3394 }
3395 }
3396}
3397
3398pub(super) struct RecoveryMonitorTicket {
3405 engine: Engine,
3406 root_session_id: SessionId,
3407}
3408
3409impl Drop for RecoveryMonitorTicket {
3410 fn drop(&mut self) {
3411 self.engine.release_recovery_claim(self.root_session_id);
3412 }
3413}
3414
3415#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3416pub(super) enum DelegationState {
3417 Queued,
3418 Starting,
3419 Running,
3420 Finished(SessionStatus),
3421}
3422
3423#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3424pub(super) struct DelegationRecord {
3425 pub(super) parent_session_id: SessionId,
3426 pub(super) parent_run_id: RunId,
3427 pub(super) parent_tool_call_id: ToolCallId,
3428 pub(super) invocation_id: InvocationId,
3429 pub(super) root_session_id: SessionId,
3430 pub(super) child_run_id: Option<RunId>,
3431 pub(super) state: DelegationState,
3432 pub(super) background: bool,
3433 pub(super) counts_slot: bool,
3434 pub(super) notification_sent: bool,
3435 pub(super) producer_id: Option<ProducerId>,
3436 pub(super) monitor_started: bool,
3437}
3438
3439struct BackgroundProducerGuard {
3440 engine: Engine,
3441 runtime: Option<tokio::runtime::Handle>,
3442 parent_session_id: SessionId,
3443 invocation_id: InvocationId,
3444 producer_id: ProducerId,
3445 active: bool,
3446}
3447
3448impl BackgroundProducerGuard {
3449 fn handoff(&mut self) {
3450 self.active = false;
3451 }
3452}
3453
3454impl Drop for BackgroundProducerGuard {
3455 fn drop(&mut self) {
3456 if !self.active {
3457 return;
3458 }
3459 let Some(runtime) = self
3460 .runtime
3461 .clone()
3462 .or_else(|| tokio::runtime::Handle::try_current().ok())
3463 else {
3464 return;
3465 };
3466 let engine = self.engine.clone();
3467 let unregister_engine = engine.clone();
3468 let session_id = self.parent_session_id;
3469 let authority = delegation_producer_authority(self.invocation_id);
3470 let producer_id = self.producer_id;
3471 let _ = engine.spawn_admission_task(&runtime, async move {
3472 let _ = unregister_engine
3473 .unregister_producer(session_id, authority, producer_id)
3474 .await;
3475 });
3476 }
3477}
3478
3479fn delegation_producer_authority(invocation_id: InvocationId) -> ProducerAuthority {
3480 ProducerAuthority {
3481 owner: ProducerOwner::Delegation { invocation_id },
3482 connection_epoch: None,
3483 }
3484}
3485
3486pub(crate) fn render_delegate_teaser_body(
3492 preview: &str,
3493 status: SessionStatus,
3494 total_lines: u64,
3495 handle: &str,
3496) -> String {
3497 let more = if preview_is_truncated(preview, total_lines) {
3500 format!("; use get_subagent_result with session_id \"{handle}\" for the full output")
3501 } else {
3502 "; full output shown".to_owned()
3503 };
3504 let preview = crate::tool_api::sanitize_tool_display(preview, PREVIEW_MAX_BYTES);
3507 let preview = if preview.trim().is_empty() {
3508 "(no output)"
3509 } else {
3510 preview.as_str()
3511 };
3512 format!(
3513 "{preview}\n\n[subagent session {handle}; {}; {total_lines} lines{more}]",
3514 session_status_name(status),
3515 )
3516}
3517
3518fn preview_is_truncated(preview: &str, total_lines: u64) -> bool {
3523 let preview_lines = if preview.is_empty() {
3524 0
3525 } else {
3526 preview.split('\n').count() as u64
3529 };
3530 preview_lines < total_lines || preview.len() > PREVIEW_MAX_BYTES - 4
3532}
3533
3534pub(crate) fn render_subagent_notification(
3537 preview: &str,
3538 status: SessionStatus,
3539 total_lines: u64,
3540 handle: &str,
3541) -> String {
3542 format!(
3543 "<subagent_notification>{}</subagent_notification>",
3544 render_delegate_teaser_body(preview, status, total_lines, handle)
3545 )
3546}
3547
3548pub(super) fn render_background_completion(teaser: &DelegateTeaser) -> String {
3549 render_subagent_notification(
3550 &teaser.preview,
3551 teaser.status,
3552 teaser.total_lines,
3553 &teaser.short_id,
3554 )
3555}
3556
3557pub(crate) fn freeze_delegated_child_policy(
3558 child_agent: &crate::runtime_snapshot::ResolvedAgent,
3559 parent_policy: &FrozenRunPolicy,
3560 child_selection: &cookie_agent_protocol::ModelSelection,
3561 inherited_suffix: &[cookie_agent_protocol::FrozenModelBinding],
3562 inherited_depth_ceiling: u32,
3563 result_limits: policy::ResultLimits,
3564) -> Result<FrozenRunPolicy, EngineError> {
3565 freeze_delegated_agent_policy(
3566 child_agent,
3567 Arc::clone(&parent_policy.registry),
3568 Arc::clone(&parent_policy.runtime),
3569 child_selection,
3570 inherited_suffix,
3571 inherited_depth_ceiling,
3572 policy::FreezeOptions {
3573 result_limits,
3574 model_retry: parent_policy.model_retry,
3575 inherited_cache_strategies: Some(
3576 inherited_suffix
3577 .iter()
3578 .map(|binding| parent_policy.raw_cache_strategy(binding))
3579 .collect(),
3580 ),
3581 },
3582 )
3583}
3584
3585pub(super) fn delegate_client_run_id(
3586 invocation_id: InvocationId,
3587) -> cookie_agent_protocol::ClientRunId {
3588 cookie_agent_protocol::ClientRunId::new(format!("delegate:{invocation_id}"))
3589 .expect("bounded delegate client run id")
3590}
3591
3592pub(super) fn render_delegate_input(request: &DelegateRequestPayload) -> String {
3593 request.prompt.clone()
3594}
3595
3596const DELEGATED_CONTEXT_MAX_BYTES: usize = 64 * 1024;
3597
3598fn context_seed_from_history(history: Vec<oven_sdk::HistoryTurn>) -> Vec<DelegatedContextTurn> {
3599 let mut turns = history
3600 .into_iter()
3601 .filter_map(|turn| match turn {
3602 oven_sdk::HistoryTurn::User(message) => {
3603 let text = message
3604 .content
3605 .into_iter()
3606 .filter_map(|part| match part {
3607 oven_sdk::InputPart::Text(text) => Some(text.text),
3608 _ => None,
3609 })
3610 .collect::<String>();
3611 (!text.is_empty()).then_some(DelegatedContextTurn {
3612 role: DelegatedContextRole::User,
3613 text,
3614 })
3615 }
3616 oven_sdk::HistoryTurn::Assistant(turn) => {
3617 let text = turn
3618 .message
3619 .content
3620 .into_iter()
3621 .filter_map(|part| match part {
3622 oven_sdk::AssistantPart::Text(text) => Some(text.text),
3623 _ => None,
3624 })
3625 .collect::<String>();
3626 (!text.is_empty()).then_some(DelegatedContextTurn {
3627 role: DelegatedContextRole::Assistant,
3628 text,
3629 })
3630 }
3631 oven_sdk::HistoryTurn::System(_) | oven_sdk::HistoryTurn::Tool(_) => None,
3632 })
3633 .collect::<Vec<_>>();
3634 let mut excess = turns
3635 .iter()
3636 .map(|turn| turn.text.len())
3637 .sum::<usize>()
3638 .saturating_sub(DELEGATED_CONTEXT_MAX_BYTES);
3639 let mut retained_from = 0;
3640 while retained_from < turns.len() && turns[retained_from].text.len() <= excess {
3641 excess -= turns[retained_from].text.len();
3642 retained_from += 1;
3643 }
3644 turns.drain(..retained_from);
3645 if excess > 0
3646 && let Some(turn) = turns.first_mut()
3647 {
3648 let mut boundary = excess;
3649 while !turn.text.is_char_boundary(boundary) {
3650 boundary += 1;
3651 }
3652 turn.text.drain(..boundary);
3653 }
3654 turns
3655}
3656
3657#[derive(Clone)]
3658pub(super) struct DelegateTeaser {
3659 pub(super) session_id: SessionId,
3660 pub(super) short_id: String,
3662 pub(super) status: SessionStatus,
3663 pub(super) preview: String,
3664 pub(super) total_lines: u64,
3665}
3666
3667fn delegate_teaser(
3668 child: &session::SessionProjection,
3669 child_session_id: SessionId,
3670 status: SessionStatus,
3671 child_run_id: Option<RunId>,
3672) -> DelegateTeaser {
3673 let text = delegate_final_text(child, child_run_id);
3674 DelegateTeaser {
3675 session_id: child_session_id,
3676 short_id: child
3677 .meta
3678 .short_id
3679 .clone()
3680 .unwrap_or_else(|| child_session_id.to_string()),
3681 status,
3682 preview: preview_text(text),
3683 total_lines: text.lines().count() as u64,
3684 }
3685}
3686
3687fn delegate_final_text(child: &session::SessionProjection, child_run_id: Option<RunId>) -> &str {
3688 child_run_id
3689 .and_then(|run_id| child.runs.get(&run_id))
3690 .and_then(|run| run.final_text.as_deref())
3691 .unwrap_or("")
3692}
3693
3694fn last_assistant_message(child: &session::SessionProjection) -> String {
3698 let events = child.log.event_snapshot();
3699 if let Some(text) = events.iter().rev().find_map(|event| match &event.payload {
3700 Event::RunCompleted {
3701 final_text: Some(text),
3702 } => Some(text.clone()),
3703 _ => None,
3704 }) {
3705 return text;
3706 }
3707 events
3708 .iter()
3709 .rev()
3710 .find_map(|event| match &event.payload {
3711 Event::ModelTurnCommitted { turn, .. } => {
3712 let text = turn
3713 .content
3714 .iter()
3715 .filter_map(|part| match part {
3716 PersistedAssistantPart::Text { text, .. } => Some(text.as_str()),
3717 _ => None,
3718 })
3719 .collect::<String>();
3720 (!text.is_empty()).then_some(text)
3721 }
3722 _ => None,
3723 })
3724 .unwrap_or_default()
3725}
3726
3727const PREVIEW_MAX_LINES: usize = 20;
3728const PREVIEW_MAX_BYTES: usize = 2048;
3729
3730fn preview_text(text: &str) -> String {
3731 let first_lines = text
3732 .lines()
3733 .take(PREVIEW_MAX_LINES)
3734 .collect::<Vec<_>>()
3735 .join("\n");
3736 if first_lines.len() <= PREVIEW_MAX_BYTES {
3737 return first_lines;
3738 }
3739 let mut end = PREVIEW_MAX_BYTES;
3740 while !first_lines.is_char_boundary(end) {
3741 end -= 1;
3742 }
3743 first_lines[..end].to_owned()
3744}
3745
3746pub(super) const fn session_status_name(status: SessionStatus) -> &'static str {
3747 match status {
3748 SessionStatus::Idle => "running",
3749 SessionStatus::Running => "running",
3750 SessionStatus::Completed => "completed",
3751 SessionStatus::Failed => "failed",
3752 SessionStatus::Cancelled => "cancelled",
3753 SessionStatus::Interrupted => "interrupted",
3754 }
3755}
3756
3757const fn background_queue_limit_reached(limit: u32, queued: u32) -> bool {
3758 queued >= limit.saturating_mul(4)
3759}
3760
3761fn paginated_subagent_result(status: &str, text: &str, offset: u32, limit: u32) -> ToolResult {
3762 let lines = text.lines().collect::<Vec<_>>();
3763 let mut output = format!("<status>{status}</status>\n<content>\n");
3764 for (index, line) in lines
3765 .iter()
3766 .enumerate()
3767 .skip(offset as usize)
3768 .take(limit as usize)
3769 {
3770 output.push_str(&format!("{}: {line}\n", index + 1));
3771 }
3772 output.push_str("</content>");
3773 ToolResult {
3774 display: Some(crate::tool_api::bounded_tool_display(&output)),
3775 retained_output: None,
3776 title: safe_display("Read subagent result"),
3777 output,
3778 metadata: serde_json::json!({
3779 "offset": offset,
3780 "limit": limit,
3781 "total_lines": lines.len(),
3782 }),
3783 truncation: None,
3784 attachments: Vec::new(),
3785 additional_messages: Vec::new(),
3786 }
3787}
3788
3789fn reference_label(session_id: Option<SessionId>, short_id: Option<&str>) -> Option<String> {
3792 short_id
3793 .map(str::to_owned)
3794 .or_else(|| session_id.map(|id| id.to_string()))
3795}
3796
3797fn steered_delegate_result(
3798 child_session_id: SessionId,
3799 short_id: Option<&str>,
3800 status: &str,
3801) -> ToolResult {
3802 let label = reference_label(Some(child_session_id), short_id)
3803 .expect("steer result has a session reference");
3804 structured_delegate_result(
3805 "Subagent steered",
3806 format!(
3807 "Subagent steered. [subagent session {label}; {}]",
3808 safe_display(status)
3809 ),
3810 serde_json::json!({
3811 "session_id": child_session_id,
3812 "handle": short_id,
3813 "status": status,
3814 }),
3815 )
3816}
3817
3818pub(crate) fn completed_delegate_result(
3819 child: &session::SessionProjection,
3820 child_run_id: Option<RunId>,
3821) -> ToolResult {
3822 terminal_delegate_result(child, child_run_id, SessionStatus::Completed)
3823}
3824
3825fn terminal_delegate_result(
3826 child: &session::SessionProjection,
3827 child_run_id: Option<RunId>,
3828 status: SessionStatus,
3829) -> ToolResult {
3830 let teaser = delegate_teaser(child, child.meta.session_id, status, child_run_id);
3831 let output = render_delegate_teaser_body(
3832 &teaser.preview,
3833 teaser.status,
3834 teaser.total_lines,
3835 &teaser.short_id,
3836 );
3837 let metadata = serde_json::json!({
3838 "session_id": teaser.session_id,
3839 "handle": teaser.short_id,
3840 "status": session_status_name(teaser.status),
3841 "total_lines": teaser.total_lines,
3842 });
3843 structured_delegate_result("Subagent finished", output, metadata)
3844}
3845
3846fn structured_delegate_result(title: &str, output: String, metadata: Value) -> ToolResult {
3847 ToolResult {
3848 display: Some(crate::tool_api::bounded_tool_display(&output)),
3849 retained_output: None,
3850 title: safe_display(title),
3851 output,
3852 metadata,
3853 truncation: None,
3854 attachments: Vec::new(),
3855 additional_messages: Vec::new(),
3856 }
3857}
3858
3859pub(super) fn cancelled_delegate_result(
3860 child_session_id: SessionId,
3861 short_id: Option<&str>,
3862 partial_report: Option<String>,
3863) -> ToolResult {
3864 let label = reference_label(Some(child_session_id), short_id)
3865 .expect("cancel result has a session reference");
3866 let mut output = format!("Delegate cancelled. [subagent session {label}]");
3867 if let Some(report) = &partial_report {
3868 output.push_str(&format!(" Partial report: {}", safe_display(report)));
3869 }
3870 structured_delegate_result(
3871 "Delegate cancelled",
3872 output,
3873 serde_json::json!({
3874 "status": "cancelled",
3875 "session_id": child_session_id,
3876 "handle": short_id,
3877 "partial_report": partial_report,
3878 }),
3879 )
3880}
3881
3882pub(super) fn cancelled_delegate_result_with_reason(
3883 child_session_id: Option<SessionId>,
3884 short_id: Option<&str>,
3885 reason: &str,
3886) -> ToolResult {
3887 let mut output = format!("Delegate cancelled: {}.", safe_display(reason));
3888 if let Some(label) = reference_label(child_session_id, short_id) {
3889 output.push_str(&format!(" [subagent session {label}]"));
3890 }
3891 structured_delegate_result(
3892 "Delegate cancelled",
3893 output,
3894 serde_json::json!({
3895 "status": "cancelled",
3896 "session_id": child_session_id,
3897 "handle": short_id,
3898 "reason": reason,
3899 }),
3900 )
3901}
3902
3903pub(super) fn delegate_failure_result(
3904 child_session_id: Option<SessionId>,
3905 short_id: Option<&str>,
3906 reason: &str,
3907) -> ToolResult {
3908 let mut output = format!("Delegate failed: {}.", safe_display(reason));
3909 if let Some(label) = reference_label(child_session_id, short_id) {
3910 output.push_str(&format!(" [subagent session {label}]"));
3911 }
3912 structured_delegate_result(
3913 "Delegate failed",
3914 output,
3915 serde_json::json!({
3916 "status": "failed",
3917 "session_id": child_session_id,
3918 "handle": short_id,
3919 "reason": reason,
3920 }),
3921 )
3922}
3923
3924fn validate_redelivery_mode(
3925 durable_background: bool,
3926 redelivery_background: bool,
3927) -> Result<(), EngineError> {
3928 if durable_background != redelivery_background {
3929 let durable = if durable_background {
3930 "background"
3931 } else {
3932 "foreground"
3933 };
3934 let redelivery = if redelivery_background {
3935 "background"
3936 } else {
3937 "foreground"
3938 };
3939 return Err(EngineError::ToolFailed(format!(
3940 "delegate redelivery execution mode conflict: durable invocation is {durable}, redelivery requested {redelivery}"
3941 )));
3942 }
3943 Ok(())
3944}
3945
3946pub(super) fn is_delegation_event_append_failure(error: &EngineError) -> bool {
3947 matches!(
3948 error,
3949 EngineError::DelegationEvents(DelegationEventError::Session(
3950 crate::session::SessionError::Event(_)
3951 )) | EngineError::ActorStopped
3952 )
3953}
3954
3955#[cfg(test)]
3956mod concurrency_tests {
3957 use super::{
3958 DELEGATED_CONTEXT_MAX_BYTES, DelegateTeaser, background_queue_limit_reached,
3959 cancelled_delegate_result, cancelled_delegate_result_with_reason,
3960 context_seed_from_history, delegate_failure_result, paginated_subagent_result,
3961 preview_is_truncated, preview_text, render_background_completion,
3962 render_delegate_teaser_body, steered_delegate_result, validate_redelivery_mode,
3963 };
3964
3965 #[test]
3966 fn delegate_outputs_are_concise_and_preserve_metadata() {
3967 let session_id = cookie_agent_protocol::SessionId::new_v7();
3968 let cases = [
3969 (
3970 steered_delegate_result(session_id, None, "running"),
3971 format!("Subagent steered. [subagent session {session_id}; running]"),
3972 serde_json::json!({"session_id": session_id, "handle": null, "status": "running"}),
3973 ),
3974 (
3975 cancelled_delegate_result(session_id, None, Some("Work so far".into())),
3976 format!(
3977 "Delegate cancelled. [subagent session {session_id}] Partial report: Work so far"
3978 ),
3979 serde_json::json!({"session_id": session_id, "handle": null, "status": "cancelled", "partial_report": "Work so far"}),
3980 ),
3981 (
3982 cancelled_delegate_result(session_id, None, None),
3983 format!("Delegate cancelled. [subagent session {session_id}]"),
3984 serde_json::json!({"session_id": session_id, "handle": null, "status": "cancelled", "partial_report": null}),
3985 ),
3986 (
3987 cancelled_delegate_result_with_reason(Some(session_id), None, "Parent stopped"),
3988 format!("Delegate cancelled: Parent stopped. [subagent session {session_id}]"),
3989 serde_json::json!({"session_id": session_id, "handle": null, "status": "cancelled", "reason": "Parent stopped"}),
3990 ),
3991 (
3992 cancelled_delegate_result_with_reason(None, None, "Parent stopped"),
3993 "Delegate cancelled: Parent stopped.".into(),
3994 serde_json::json!({"session_id": null, "handle": null, "status": "cancelled", "reason": "Parent stopped"}),
3995 ),
3996 (
3997 delegate_failure_result(Some(session_id), None, "Child unavailable"),
3998 format!("Delegate failed: Child unavailable. [subagent session {session_id}]"),
3999 serde_json::json!({"session_id": session_id, "handle": null, "status": "failed", "reason": "Child unavailable"}),
4000 ),
4001 (
4002 delegate_failure_result(None, None, "Child unavailable"),
4003 "Delegate failed: Child unavailable.".into(),
4004 serde_json::json!({"session_id": null, "handle": null, "status": "failed", "reason": "Child unavailable"}),
4005 ),
4006 ];
4007 for (result, output, metadata) in cases {
4008 assert_eq!(result.output, output);
4009 assert_eq!(result.metadata, metadata);
4010 assert!(serde_json::from_str::<serde_json::Value>(&result.output).is_err());
4011 assert!(result.truncation.is_none());
4012 assert!(result.attachments.is_empty());
4013 assert!(result.additional_messages.is_empty());
4014 }
4015 }
4016
4017 #[test]
4018 fn delegate_outputs_surface_the_handle_in_place_of_the_uuid() {
4019 let session_id = cookie_agent_protocol::SessionId::new_v7();
4020 let handle = "explore_1a2b3c4d";
4021 let steered = steered_delegate_result(session_id, Some(handle), "running");
4022 assert_eq!(
4023 steered.output,
4024 "Subagent steered. [subagent session explore_1a2b3c4d; running]"
4025 );
4026 let cancelled = cancelled_delegate_result(session_id, Some(handle), None);
4027 assert_eq!(
4028 cancelled.output,
4029 "Delegate cancelled. [subagent session explore_1a2b3c4d]"
4030 );
4031 assert_eq!(
4032 cancelled.metadata["session_id"],
4033 serde_json::json!(session_id)
4034 );
4035 assert_eq!(cancelled.metadata["handle"], serde_json::json!(handle));
4036 let failed = delegate_failure_result(Some(session_id), Some(handle), "Child unavailable");
4037 assert_eq!(
4038 failed.output,
4039 "Delegate failed: Child unavailable. [subagent session explore_1a2b3c4d]"
4040 );
4041 assert!(!failed.output.contains(&session_id.to_string()));
4042 }
4043
4044 #[test]
4045 fn background_notification_snapshot_matches_the_foreground_teaser_body() {
4046 let teaser = DelegateTeaser {
4047 session_id: cookie_agent_protocol::SessionId::new_v7(),
4048 short_id: "explore_1a2b3c4d".into(),
4049 status: cookie_agent_protocol::SessionStatus::Completed,
4050 preview: "first line\nsecond line".into(),
4051 total_lines: 79,
4052 };
4053 insta::assert_snapshot!(
4055 render_background_completion(&teaser),
4056 @r###"
4057<subagent_notification>first line
4058second line
4059
4060[subagent session explore_1a2b3c4d; completed; 79 lines; use get_subagent_result with session_id "explore_1a2b3c4d" for the full output]</subagent_notification>
4061"###
4062 );
4063 }
4064
4065 #[test]
4066 fn foreground_tool_result_text_snapshot_matches_the_shared_teaser_body() {
4067 insta::assert_snapshot!(
4070 render_delegate_teaser_body(
4071 "first line\nsecond line",
4072 cookie_agent_protocol::SessionStatus::Completed,
4073 79,
4074 "explore_1a2b3c4d",
4075 ),
4076 @r###"
4077first line
4078second line
4079
4080[subagent session explore_1a2b3c4d; completed; 79 lines; use get_subagent_result with session_id "explore_1a2b3c4d" for the full output]
4081"###
4082 );
4083 }
4084
4085 #[test]
4086 fn delegate_display_fields_are_bounded_and_control_free_without_changing_metadata() {
4087 let session_id = cookie_agent_protocol::SessionId::new_v7();
4088 let text = format!("\x1b\n\t{}", "é".repeat(2048));
4089 for (result, key) in [
4090 (steered_delegate_result(session_id, None, &text), "status"),
4091 (
4092 cancelled_delegate_result(session_id, None, Some(text.clone())),
4093 "partial_report",
4094 ),
4095 (
4096 cancelled_delegate_result_with_reason(Some(session_id), None, &text),
4097 "reason",
4098 ),
4099 (
4100 delegate_failure_result(Some(session_id), None, &text),
4101 "reason",
4102 ),
4103 ] {
4104 assert_eq!(result.metadata[key], text);
4105 assert!(result.output.len() < 1200);
4106 assert!(!result.output.chars().any(char::is_control));
4107 assert_eq!(result.output.matches(&session_id.to_string()).count(), 1);
4108 }
4109 }
4110
4111 #[test]
4112 fn background_queue_rejects_only_at_four_times_concurrency() {
4113 assert!(!background_queue_limit_reached(4, 15));
4114 assert!(background_queue_limit_reached(4, 16));
4115 assert!(background_queue_limit_reached(4, 17));
4116 }
4117
4118 #[test]
4119 fn redelivery_execution_mode_must_match_in_both_directions() {
4120 assert!(validate_redelivery_mode(false, false).is_ok());
4121 assert!(validate_redelivery_mode(true, true).is_ok());
4122 let foreground_to_background = validate_redelivery_mode(false, true)
4123 .expect_err("foreground to background must fail")
4124 .to_string();
4125 assert!(foreground_to_background.contains("durable invocation is foreground"));
4126 assert!(foreground_to_background.contains("requested background"));
4127 let background_to_foreground = validate_redelivery_mode(true, false)
4128 .expect_err("background to foreground must fail")
4129 .to_string();
4130 assert!(background_to_foreground.contains("durable invocation is background"));
4131 assert!(background_to_foreground.contains("requested foreground"));
4132 }
4133
4134 #[test]
4135 fn complete_preview_drops_the_full_output_hint() {
4136 insta::assert_snapshot!(
4137 render_delegate_teaser_body(
4138 "first line\nsecond line",
4139 cookie_agent_protocol::SessionStatus::Completed,
4140 2,
4141 "explore_1a2b3c4d",
4142 ),
4143 @r###"
4144first line
4145second line
4146
4147[subagent session explore_1a2b3c4d; completed; 2 lines; full output shown]
4148"###
4149 );
4150 }
4151
4152 #[test]
4153 fn preview_keeps_its_shape_but_not_other_control_characters() {
4154 let body = render_delegate_teaser_body(
4155 "- item\n\tindented\x1b[31m red\x07",
4156 cookie_agent_protocol::SessionStatus::Completed,
4157 2,
4158 "explore_1a2b3c4d",
4159 );
4160 assert!(body.starts_with("- item\n\tindented [31m red \n\n[subagent session"));
4161 let empty = render_delegate_teaser_body(
4162 "",
4163 cookie_agent_protocol::SessionStatus::Completed,
4164 0,
4165 "explore_1a2b3c4d",
4166 );
4167 assert!(empty.starts_with("(no output)\n\n[subagent session"));
4168 }
4169
4170 #[test]
4171 fn truncation_is_judged_from_the_preview_the_output_produced() {
4172 let cases = [
4173 ("", false),
4174 ("one line", false),
4175 ("trailing blank\n\n", false),
4176 ("two\nlines", false),
4177 ]
4178 .into_iter()
4179 .map(|(text, cut)| (text.to_owned(), cut))
4180 .chain([
4181 (vec!["line"; 21].join("\n"), true),
4183 (vec!["line"; 20].join("\n"), false),
4184 ("é".repeat(1100), true),
4186 (format!("a{}", "€".repeat(700)), true),
4188 ]);
4189 for (text, cut) in cases {
4190 let preview = preview_text(&text);
4191 let total_lines = text.lines().count() as u64;
4192 assert_eq!(
4193 preview_is_truncated(&preview, total_lines),
4194 cut,
4195 "{} bytes, {total_lines} lines",
4196 text.len()
4197 );
4198 }
4199 }
4200
4201 #[test]
4202 fn preview_is_line_and_utf8_bounded() {
4203 let text = (0..30)
4204 .map(|index| format!("{index}: {}", "é".repeat(200)))
4205 .collect::<Vec<_>>()
4206 .join("\n");
4207 let preview = preview_text(&text);
4208 assert!(preview.lines().count() <= 20);
4209 assert!(preview.len() <= 2048);
4210 assert!(std::str::from_utf8(preview.as_bytes()).is_ok());
4211 }
4212
4213 #[test]
4214 fn result_pages_match_read_line_numbering_and_empty_running_shape() {
4215 let page = paginated_subagent_result("completed", "one\ntwo\nthree", 1, 1);
4216 assert_eq!(
4217 page.output,
4218 "<status>completed</status>\n<content>\n2: two\n</content>"
4219 );
4220 assert_eq!(page.metadata["total_lines"], 3);
4221
4222 let running = paginated_subagent_result("running", "", 0, 20);
4223 assert_eq!(
4224 running.output,
4225 "<status>running</status>\n<content>\n</content>"
4226 );
4227 assert_eq!(running.metadata["total_lines"], 0);
4228 }
4229
4230 #[test]
4231 fn inherited_context_is_text_only_and_truncates_oldest_bytes_first() {
4232 let history = vec![
4233 oven_sdk::HistoryTurn::system(oven_sdk::SystemMessage::new(vec![
4234 oven_sdk::SystemPart::Text(oven_sdk::TextPart::new("private system prompt")),
4235 ])),
4236 oven_sdk::HistoryTurn::user(oven_sdk::UserMessage::new(vec![
4237 oven_sdk::InputPart::Text(oven_sdk::TextPart::new("old-user".repeat(125))),
4238 ])),
4239 oven_sdk::HistoryTurn::tool(oven_sdk::ToolMessage::new(vec![
4240 oven_sdk::ToolResultPart::new(
4241 "call",
4242 oven_sdk::ToolContent::Text("secret tool output".into()),
4243 ),
4244 ])),
4245 oven_sdk::HistoryTurn::assistant(oven_sdk::CompletedTurn::new(
4246 oven_sdk::AssistantMessage::new(vec![
4247 oven_sdk::AssistantPart::Text(oven_sdk::TextPart::new("n".repeat(70_000))),
4248 oven_sdk::AssistantPart::ToolResult(oven_sdk::ToolResultPart::new(
4249 "hosted",
4250 oven_sdk::ToolContent::Text("hosted output".into()),
4251 )),
4252 ]),
4253 oven_sdk::Finish::new(oven_sdk::Usage::default(), oven_sdk::FinishReason::Stop),
4254 )),
4255 ];
4256 let seed = context_seed_from_history(history);
4257 assert_eq!(seed.len(), 1);
4258 assert_eq!(
4259 seed[0].role,
4260 cookie_agent_protocol::DelegatedContextRole::Assistant
4261 );
4262 assert_eq!(seed[0].text.len(), DELEGATED_CONTEXT_MAX_BYTES);
4263 assert!(!seed[0].text.contains("old-user"));
4264 assert!(!seed[0].text.contains("tool output"));
4265 }
4266}