1use crate::agent_resource::Capability;
8use anyhow::{ensure, Context, Result};
9use chrono::Utc;
10use serde::{Deserialize, Serialize};
11use serde_json::{json, Value};
12use sqlx::{FromRow, Sqlite, SqlitePool, Transaction};
13use uuid::Uuid;
14
15#[derive(Debug)]
16pub struct AccountingOverflow;
17
18impl std::fmt::Display for AccountingOverflow {
19 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
20 formatter.write_str("run accounting overflow")
21 }
22}
23
24impl std::error::Error for AccountingOverflow {}
25
26#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
27#[serde(default, deny_unknown_fields)]
28pub struct RunBudgets {
29 pub max_turns: Option<u64>,
30 pub timeout_seconds: Option<u64>,
31 pub max_total_tokens: Option<u64>,
32 pub max_tool_calls: Option<u64>,
33 #[serde(skip_serializing_if = "Option::is_none")]
34 pub max_context_bytes: Option<u64>,
35}
36
37impl RunBudgets {
38 fn validate(&self) -> Result<()> {
39 ensure!(
40 self.max_turns != Some(0)
41 && self.timeout_seconds != Some(0)
42 && self.max_total_tokens != Some(0)
43 && self.max_tool_calls != Some(0)
44 && self.max_context_bytes != Some(0),
45 "run budgets must be positive when configured"
46 );
47 Ok(())
48 }
49}
50
51#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum TokenAccounting {
54 Complete,
55 Partial,
56 #[default]
57 Unavailable,
58}
59
60#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
61pub struct RunUsage {
62 pub model_turns: u64,
63 pub input_tokens: Option<u64>,
64 pub output_tokens: Option<u64>,
65 pub total_tokens: Option<u64>,
66 pub responses_with_usage: u64,
67 pub responses_without_usage: u64,
68 pub tool_calls: u64,
69 pub token_accounting: TokenAccounting,
70}
71
72#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
78#[serde(rename_all = "snake_case")]
79pub enum RecoveryDisposition {
80 Complete,
81 ResumeEligible,
82 Suspended,
83 #[default]
84 RestartRequired,
85 ReconciliationRequired,
86}
87
88#[derive(Debug, Serialize, FromRow)]
90pub struct AgentRun {
91 pub id: String,
92 pub workspace_id: String,
93 pub agent_name: String,
94 pub agent_version: String,
95 pub model: String,
96 pub status: String,
97 pub lifecycle: RunLifecycle,
98 pub outcome: Option<RunOutcomeKind>,
99 pub reason_code: Option<String>,
100 pub reason_detail: Option<sqlx::types::Json<Value>>,
101 pub created_at: i64,
102 pub updated_at: i64,
103 pub completed_at: Option<i64>,
104 pub input: String,
105 pub output: Option<String>,
106 pub error: Option<String>,
107 pub last_sequence: i64,
108 pub budgets: sqlx::types::Json<RunBudgets>,
109 pub usage: sqlx::types::Json<RunUsage>,
110 pub suspended_ms: i64,
111 pub elapsed_ms: i64,
112 pub execution_elapsed_ms: i64,
113 #[sqlx(skip)]
114 pub recovery_disposition: RecoveryDisposition,
115}
116
117#[derive(Debug, Serialize, FromRow)]
119pub struct RunLineage {
120 pub run_id: String,
121 pub parent_run_id: Option<String>,
122 pub root_run_id: String,
123 pub depth: i64,
124 pub parent_call_id: Option<String>,
125}
126
127#[derive(Debug, Serialize, FromRow)]
129pub struct AgentEvent {
130 pub run_id: String,
131 pub sequence: i64,
132 pub timestamp: i64,
133 pub event_type: String,
134 pub payload: sqlx::types::Json<Value>,
135}
136
137pub const WORKING_STATE_VERSION: i64 = 1;
138const MAX_WORKING_STATE_BYTES: usize = 256 * 1024;
139
140#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
141#[serde(rename_all = "snake_case")]
142pub enum WorkingStateStatus {
143 Current,
144 Stale,
145 Unavailable,
146 Unsupported,
147}
148
149#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
150#[serde(deny_unknown_fields)]
151pub struct WorkingStateSnapshot {
152 pub objective: WorkingStateObjective,
153 pub run: WorkingStateRun,
154 pub usage: RunUsage,
155 pub latest_event: WorkingStateEventReference,
156 pub artifacts: WorkingStateArtifacts,
157}
158
159#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
160#[serde(deny_unknown_fields)]
161pub struct WorkingStateObjective {
162 pub kind: String,
163 pub run_id: String,
164}
165
166#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
167#[serde(deny_unknown_fields)]
168pub struct WorkingStateRun {
169 pub lifecycle: RunLifecycle,
170 pub outcome: Option<RunOutcomeKind>,
171 pub reason_code: Option<String>,
172}
173
174#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
175#[serde(deny_unknown_fields)]
176pub struct WorkingStateEventReference {
177 pub sequence: i64,
178 pub event_type: String,
179}
180
181#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
182#[serde(deny_unknown_fields)]
183pub struct WorkingStateArtifacts {
184 pub items: Vec<ArtifactMetadata>,
185 pub total: i64,
186 pub omitted: i64,
187}
188
189#[derive(Debug, Serialize)]
190pub struct WorkingStateInspection {
191 pub status: WorkingStateStatus,
192 pub projection_version: Option<i64>,
193 pub revision: Option<i64>,
194 pub event_cursor: Option<i64>,
195 pub updated_at: Option<i64>,
196 pub snapshot: Option<WorkingStateSnapshot>,
197}
198
199#[derive(FromRow)]
200struct WorkingStateRow {
201 projection_version: i64,
202 revision: i64,
203 event_cursor: i64,
204 updated_at: i64,
205 snapshot: sqlx::types::Json<Value>,
206}
207
208#[derive(Debug, Serialize, FromRow)]
211pub struct AgentCheckpoint {
212 pub run_id: String,
213 pub sequence: i64,
214 pub schema_version: i64,
215 pub turn: i64,
216 pub created_at: i64,
217 pub state: sqlx::types::Json<Value>,
218}
219
220#[derive(Debug, Clone, FromRow)]
221pub struct RunSuspension {
222 pub run_id: String,
223 pub suspension_sequence: i64,
224 pub outcome: RunOutcomeKind,
225 pub reason_code: String,
226 pub prompt: String,
227 pub assistant_message: Option<sqlx::types::Json<Value>>,
228 pub continuation_input: Option<String>,
229 pub continued_at: Option<i64>,
230}
231
232#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, sqlx::Type)]
233#[serde(rename_all = "snake_case")]
234#[sqlx(type_name = "TEXT", rename_all = "snake_case")]
235pub enum RunLifecycle {
236 Active,
237 Suspended,
238 Terminal,
239}
240
241#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, sqlx::Type)]
242#[serde(rename_all = "snake_case")]
243#[sqlx(type_name = "TEXT", rename_all = "snake_case")]
244pub enum RunOutcomeKind {
245 Completed,
246 Blocked,
247 NeedsUserInput,
248 Failed,
249 LimitExceeded,
250 Cancelled,
251}
252
253#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
254#[serde(rename_all = "snake_case")]
255pub enum BlockedReason {
256 ExternalDependency,
257 EnvironmentUnavailable,
258 PolicyRestriction,
259 ResourceUnavailable,
260}
261
262#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
263#[serde(rename_all = "snake_case")]
264pub enum NeedsUserInputReason {
265 DecisionRequired,
266 InformationRequired,
267}
268
269#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
270#[serde(rename_all = "snake_case")]
271pub enum FailureReason {
272 ModelError,
273 ModelRefusal,
274 ModelOutputTruncated,
275 ContentFiltered,
276 InvalidModelResponse,
277 ToolError,
278 PermissionDenied,
279 StorageError,
280 AccountingError,
281 RecoveryRejected,
282 ToolLoopDetected,
283 LegacyFailure,
284}
285
286#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
287#[serde(rename_all = "snake_case")]
288pub enum LimitReason {
289 ModelTurns,
290 Duration,
291 TotalTokens,
292 TokenUsageUnavailable,
293 ToolCalls,
294 ToolCallsPerTurn,
295 CheckpointSize,
296 OutputSize,
297 ContextBytes,
298}
299
300#[derive(Debug, Clone, Copy, PartialEq, Eq)]
301pub enum BudgetDecision {
302 Allowed,
303 Exceeded(LimitReason),
304}
305
306#[derive(FromRow)]
307struct BudgetRow {
308 id: String,
309 budgets: sqlx::types::Json<RunBudgets>,
310 usage: sqlx::types::Json<RunUsage>,
311}
312
313#[derive(FromRow)]
314struct ContinuationRunRow {
315 created_at: i64,
316 completed_at: i64,
317 suspended_ms: i64,
318 budgets: sqlx::types::Json<RunBudgets>,
319 usage: sqlx::types::Json<RunUsage>,
320}
321
322pub enum RunOutcome {
324 Completed(String),
325 Failed(String),
326 Cancelled,
327 Blocked {
328 code: BlockedReason,
329 message: String,
330 },
331 NeedsUserInput {
332 code: NeedsUserInputReason,
333 question: String,
334 },
335 FailedWithReason {
336 code: FailureReason,
337 error: String,
338 },
339 FailedWithDetail {
340 code: FailureReason,
341 error: String,
342 detail: Value,
343 },
344 LimitExceeded {
345 code: LimitReason,
346 },
347}
348
349struct StoppedRun {
350 lifecycle: RunLifecycle,
351 outcome: RunOutcomeKind,
352 reason_code: String,
353 reason_detail: Value,
354 status: &'static str,
355 output: Option<String>,
356 error: Option<String>,
357}
358
359fn reason_code<T: Serialize>(code: T) -> String {
360 serde_json::to_value(code)
361 .expect("reason codes serialize")
362 .as_str()
363 .expect("reason codes are strings")
364 .to_owned()
365}
366
367fn stopped(outcome: RunOutcome) -> Result<StoppedRun> {
368 let stopped = match outcome {
369 RunOutcome::Completed(output) => StoppedRun {
370 lifecycle: RunLifecycle::Terminal,
371 outcome: RunOutcomeKind::Completed,
372 reason_code: "final_response".into(),
373 reason_detail: json!({}),
374 status: "completed",
375 output: Some(output),
376 error: None,
377 },
378 RunOutcome::Failed(error) => StoppedRun {
379 lifecycle: RunLifecycle::Terminal,
380 outcome: RunOutcomeKind::Failed,
381 reason_code: "legacy_failure".into(),
382 reason_detail: json!({}),
383 status: "failed",
384 output: None,
385 error: Some(error),
386 },
387 RunOutcome::Cancelled => StoppedRun {
388 lifecycle: RunLifecycle::Terminal,
389 outcome: RunOutcomeKind::Cancelled,
390 reason_code: "cancellation_requested".into(),
391 reason_detail: json!({}),
392 status: "cancelled",
393 output: None,
394 error: None,
395 },
396 RunOutcome::Blocked { code, message } => {
397 ensure!(
398 !message.trim().is_empty() && message.len() <= 8192,
399 "invalid blocked message"
400 );
401 StoppedRun {
402 lifecycle: RunLifecycle::Suspended,
403 outcome: RunOutcomeKind::Blocked,
404 reason_code: reason_code(code),
405 reason_detail: json!({"message": message}),
406 status: "failed",
407 output: None,
408 error: None,
409 }
410 }
411 RunOutcome::NeedsUserInput { code, question } => {
412 ensure!(
413 !question.trim().is_empty() && question.len() <= 8192,
414 "invalid user-input question"
415 );
416 StoppedRun {
417 lifecycle: RunLifecycle::Suspended,
418 outcome: RunOutcomeKind::NeedsUserInput,
419 reason_code: reason_code(code),
420 reason_detail: json!({"question": question}),
421 status: "failed",
422 output: None,
423 error: None,
424 }
425 }
426 RunOutcome::FailedWithReason { code, error } => StoppedRun {
427 lifecycle: RunLifecycle::Terminal,
428 outcome: RunOutcomeKind::Failed,
429 reason_code: reason_code(code),
430 reason_detail: json!({}),
431 status: "failed",
432 output: None,
433 error: Some(error),
434 },
435 RunOutcome::FailedWithDetail {
436 code,
437 error,
438 detail,
439 } => StoppedRun {
440 lifecycle: RunLifecycle::Terminal,
441 outcome: RunOutcomeKind::Failed,
442 reason_code: reason_code(code),
443 reason_detail: detail,
444 status: "failed",
445 output: None,
446 error: Some(error),
447 },
448 RunOutcome::LimitExceeded { code } => StoppedRun {
449 lifecycle: RunLifecycle::Terminal,
450 outcome: RunOutcomeKind::LimitExceeded,
451 reason_code: reason_code(code),
452 reason_detail: json!({}),
453 status: "failed",
454 output: None,
455 error: Some(
456 match code {
457 LimitReason::ModelTurns => "maximum model turns reached",
458 LimitReason::Duration => "execution timeout",
459 LimitReason::TotalTokens => "total token budget exceeded",
460 LimitReason::TokenUsageUnavailable => "token usage unavailable",
461 LimitReason::ToolCalls => "tool call budget exceeded",
462 LimitReason::ToolCallsPerTurn => "too many tool calls in one turn",
463 LimitReason::CheckpointSize => "checkpoint size limit exceeded",
464 LimitReason::OutputSize => "output size limit exceeded",
465 LimitReason::ContextBytes => "context byte limit exceeded",
466 }
467 .into(),
468 ),
469 },
470 };
471 ensure!(
472 serde_json::to_vec(&stopped.reason_detail)?.len() <= 8192,
473 "run reason detail exceeds 8 KiB"
474 );
475 Ok(stopped)
476}
477
478fn reason_code_for_event(outcome: RunOutcomeKind) -> &'static str {
479 match outcome {
480 RunOutcomeKind::Completed => "completed",
481 RunOutcomeKind::Blocked => "blocked",
482 RunOutcomeKind::NeedsUserInput => "needs_user_input",
483 RunOutcomeKind::Failed => "failed",
484 RunOutcomeKind::LimitExceeded => "limit_exceeded",
485 RunOutcomeKind::Cancelled => "cancelled",
486 }
487}
488
489#[derive(Debug, Serialize, FromRow)]
490pub struct ToolInvocation {
491 pub run_id: String,
492 pub call_id: String,
493 pub requested_sequence: i64,
494 pub tool_name: String,
495 pub arguments: sqlx::types::Json<Value>,
496 pub status: String,
497 pub result: Option<sqlx::types::Json<Value>>,
498 pub result_content: Option<String>,
499 pub error: Option<String>,
500 pub outcome_class: Option<String>,
501 pub error_code: Option<String>,
502 pub repetition_key: Option<String>,
503 pub consecutive_count: Option<i64>,
504 pub started_at: Option<i64>,
505 pub completed_at: Option<i64>,
506}
507
508pub enum ToolOutcome {
509 Completed(Value),
510 Failed(String),
511 Denied(String),
512 Recoverable {
513 code: String,
514 message: String,
515 observation: Value,
516 observation_content: String,
517 repetition_key: String,
518 },
519 Terminal {
520 code: String,
521 message: String,
522 },
523 UncertainSideEffect {
524 code: String,
525 message: String,
526 },
527}
528
529#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
531#[serde(deny_unknown_fields)]
532pub struct ArtifactMetadata {
533 pub sequence: i64,
534 pub relative_path: String,
535 pub sha256: String,
536 pub size: u64,
537}
538
539pub const MAX_ARTIFACT_BYTES: u64 = 16 * 1024 * 1024;
540
541#[derive(Clone)]
543pub struct AgentRunStore {
544 pool: SqlitePool,
545 workspace_id: String,
546}
547
548impl AgentRunStore {
549 pub fn new(pool: SqlitePool, workspace_id: &str) -> Result<Self> {
550 ensure!(!workspace_id.trim().is_empty(), "workspace ID is required");
551 Ok(Self {
552 pool,
553 workspace_id: workspace_id.to_owned(),
554 })
555 }
556
557 pub async fn create_run(
559 &self,
560 agent_name: &str,
561 agent_version: &str,
562 model: &str,
563 input: &str,
564 ) -> Result<AgentRun> {
565 self.create_run_with_budgets(
566 agent_name,
567 agent_version,
568 model,
569 input,
570 RunBudgets::default(),
571 )
572 .await
573 }
574
575 pub async fn create_run_with_budgets(
576 &self,
577 agent_name: &str,
578 agent_version: &str,
579 model: &str,
580 input: &str,
581 budgets: RunBudgets,
582 ) -> Result<AgentRun> {
583 let mut tx = self.pool.begin().await?;
584 let id = self
585 .create_root_run_in(&mut tx, agent_name, agent_version, model, input, &budgets)
586 .await?;
587 tx.commit().await?;
588 self.get_run(&id).await?.context("created run missing")
589 }
590
591 pub(crate) async fn create_root_run_in(
594 &self,
595 tx: &mut Transaction<'_, Sqlite>,
596 agent_name: &str,
597 agent_version: &str,
598 model: &str,
599 input: &str,
600 budgets: &RunBudgets,
601 ) -> Result<String> {
602 budgets.validate()?;
603 ensure!(!agent_name.trim().is_empty(), "agent name is required");
604 ensure!(
605 !agent_version.trim().is_empty(),
606 "agent version is required"
607 );
608 ensure!(!model.trim().is_empty(), "model is required");
609 let id = Uuid::new_v4().to_string();
610 let now = Utc::now().timestamp_millis();
611 sqlx::query("INSERT INTO agent_runs (id, workspace_id, agent_name, agent_version, model, status, created_at, updated_at, input, budgets) VALUES (?, ?, ?, ?, ?, 'running', ?, ?, ?, ?)")
612 .bind(&id).bind(&self.workspace_id).bind(agent_name).bind(agent_version).bind(model).bind(now).bind(now).bind(input).bind(sqlx::types::Json(budgets))
613 .execute(&mut **tx).await?;
614 sqlx::query("INSERT INTO agent_run_lineage (run_id, root_run_id, depth) VALUES (?, ?, 0)")
615 .bind(&id)
616 .bind(&id)
617 .execute(&mut **tx)
618 .await?;
619 self.append_in(
620 tx,
621 &id,
622 "run.started",
623 &json!({"agent": agent_name, "model": model, "workspace_id": self.workspace_id, "budgets": budgets}),
624 )
625 .await?;
626 Ok(id)
627 }
628
629 pub async fn lineage(&self, id: &str) -> Result<RunLineage> {
630 if !self.has_lineage_table().await? {
631 self.get_run(id)
632 .await?
633 .context("run not found in workspace")?;
634 return Ok(RunLineage {
635 run_id: id.to_owned(),
636 parent_run_id: None,
637 root_run_id: id.to_owned(),
638 depth: 0,
639 parent_call_id: None,
640 });
641 }
642 sqlx::query_as("SELECT r.id AS run_id, l.parent_run_id, COALESCE(l.root_run_id, r.id) AS root_run_id, COALESCE(l.depth, 0) AS depth, l.parent_call_id FROM agent_runs r LEFT JOIN agent_run_lineage l ON l.run_id = r.id WHERE r.id = ? AND r.workspace_id = ?")
643 .bind(id).bind(&self.workspace_id).fetch_optional(&self.pool).await?.context("run not found in workspace")
644 }
645
646 pub async fn children(&self, id: &str) -> Result<Vec<RunLineage>> {
647 self.get_run(id)
648 .await?
649 .context("run not found in workspace")?;
650 if !self.has_lineage_table().await? {
651 return Ok(Vec::new());
652 }
653 Ok(sqlx::query_as("SELECT l.* FROM agent_run_lineage l JOIN agent_runs r ON r.id = l.run_id WHERE l.parent_run_id = ? AND r.workspace_id = ? ORDER BY r.created_at, r.id")
654 .bind(id).bind(&self.workspace_id).fetch_all(&self.pool).await?)
655 }
656
657 async fn has_lineage_table(&self) -> Result<bool> {
658 Ok(sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'agent_run_lineage')")
659 .fetch_one(&self.pool).await?)
660 }
661
662 pub async fn create_child_run(
664 &self,
665 parent_id: &str,
666 call_id: &str,
667 agent_name: &str,
668 agent_version: &str,
669 model: &str,
670 input: &str,
671 ) -> Result<AgentRun> {
672 self.create_child_run_with_budgets(
673 parent_id,
674 call_id,
675 agent_name,
676 agent_version,
677 model,
678 input,
679 RunBudgets::default(),
680 )
681 .await
682 }
683
684 #[allow(clippy::too_many_arguments)]
685 pub async fn create_child_run_with_budgets(
686 &self,
687 parent_id: &str,
688 call_id: &str,
689 agent_name: &str,
690 agent_version: &str,
691 model: &str,
692 input: &str,
693 budgets: RunBudgets,
694 ) -> Result<AgentRun> {
695 budgets.validate()?;
696 ensure!(
697 !agent_name.trim().is_empty()
698 && !agent_version.trim().is_empty()
699 && !model.trim().is_empty(),
700 "agent identity is required"
701 );
702 let mut tx = self.pool.begin().await?;
703 let found: Option<String> = sqlx::query_scalar("UPDATE agent_runs SET updated_at = updated_at WHERE id = ? AND workspace_id = ? AND status = 'running' RETURNING id")
704 .bind(parent_id).bind(&self.workspace_id).fetch_optional(&mut *tx).await?;
705 ensure!(
706 found.is_some(),
707 "parent not found in workspace or already terminal"
708 );
709 let started: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM tool_invocations WHERE run_id = ? AND call_id = ? AND tool_name = 'agent.invoke' AND status = 'started')")
710 .bind(parent_id).bind(call_id).fetch_one(&mut *tx).await?;
711 ensure!(started, "delegation invocation is not started");
712 let lineage: Option<RunLineage> =
713 sqlx::query_as("SELECT * FROM agent_run_lineage WHERE run_id = ?")
714 .bind(parent_id)
715 .fetch_optional(&mut *tx)
716 .await?;
717 let (root, depth) = lineage
718 .map(|l| (l.root_run_id, l.depth + 1))
719 .unwrap_or((parent_id.to_owned(), 1));
720 ensure!(depth <= 4, "maximum delegation depth exceeded");
721 let id = Uuid::new_v4().to_string();
722 let now = Utc::now().timestamp_millis();
723 sqlx::query("INSERT INTO agent_runs (id, workspace_id, agent_name, agent_version, model, status, created_at, updated_at, input, budgets) VALUES (?, ?, ?, ?, ?, 'running', ?, ?, ?, ?)")
724 .bind(&id).bind(&self.workspace_id).bind(agent_name).bind(agent_version).bind(model).bind(now).bind(now).bind(input).bind(sqlx::types::Json(&budgets)).execute(&mut *tx).await?;
725 sqlx::query("INSERT INTO agent_run_lineage (run_id, parent_run_id, root_run_id, depth, parent_call_id) VALUES (?, ?, ?, ?, ?)")
726 .bind(&id).bind(parent_id).bind(&root).bind(depth).bind(call_id).execute(&mut *tx).await?;
727 self.append_in(
728 &mut tx,
729 parent_id,
730 "delegation.started",
731 &json!({"call_id": call_id, "child_run_id": id}),
732 )
733 .await?;
734 self.append_in(&mut tx, &id, "run.created", &json!({"agent": agent_name, "model": model, "workspace_id": self.workspace_id, "parent_run_id": parent_id, "root_run_id": root, "depth": depth, "budgets": budgets})).await?;
735 tx.commit().await?;
736 self.get_run(&id).await?.context("created child missing")
737 }
738
739 pub async fn get_run(&self, id: &str) -> Result<Option<AgentRun>> {
740 let mut run = self.get_run_raw(id).await?;
741 if let Some(run) = &mut run {
742 run.recovery_disposition = self.classify_recovery(run).await?;
743 }
744 Ok(run)
745 }
746
747 pub(crate) async fn get_run_raw(&self, id: &str) -> Result<Option<AgentRun>> {
748 Ok(sqlx::query_as("SELECT *, MAX(0, COALESCE(completed_at, CAST(unixepoch('subsec') * 1000 AS INTEGER)) - created_at) AS elapsed_ms, MAX(0, COALESCE(completed_at, CAST(unixepoch('subsec') * 1000 AS INTEGER)) - created_at - suspended_ms) AS execution_elapsed_ms FROM agent_runs WHERE id = ? AND workspace_id = ?")
749 .bind(id)
750 .bind(&self.workspace_id)
751 .fetch_optional(&self.pool)
752 .await?)
753 }
754
755 pub async fn history(&self, limit: u32) -> Result<Vec<AgentRun>> {
757 let mut runs: Vec<AgentRun> = sqlx::query_as(
758 "SELECT *, MAX(0, COALESCE(completed_at, CAST(unixepoch('subsec') * 1000 AS INTEGER)) - created_at) AS elapsed_ms, MAX(0, COALESCE(completed_at, CAST(unixepoch('subsec') * 1000 AS INTEGER)) - created_at - suspended_ms) AS execution_elapsed_ms FROM agent_runs WHERE workspace_id = ? ORDER BY created_at DESC, id LIMIT ?",
759 )
760 .bind(&self.workspace_id)
761 .bind(limit)
762 .fetch_all(&self.pool)
763 .await?;
764 for run in &mut runs {
765 run.recovery_disposition = self.classify_recovery(run).await?;
766 }
767 Ok(runs)
768 }
769
770 pub async fn recovery_disposition(&self, id: &str) -> Result<RecoveryDisposition> {
773 let run: AgentRun = sqlx::query_as("SELECT *, MAX(0, COALESCE(completed_at, CAST(unixepoch('subsec') * 1000 AS INTEGER)) - created_at) AS elapsed_ms, MAX(0, COALESCE(completed_at, CAST(unixepoch('subsec') * 1000 AS INTEGER)) - created_at - suspended_ms) AS execution_elapsed_ms FROM agent_runs WHERE id = ? AND workspace_id = ?")
774 .bind(id)
775 .bind(&self.workspace_id)
776 .fetch_optional(&self.pool)
777 .await?
778 .context("run not found in workspace")?;
779 self.classify_recovery(&run).await
780 }
781
782 async fn classify_recovery(&self, run: &AgentRun) -> Result<RecoveryDisposition> {
783 if run.outcome == Some(RunOutcomeKind::Completed) {
784 return Ok(RecoveryDisposition::Complete);
785 }
786 if run.lifecycle == RunLifecycle::Suspended {
787 return Ok(RecoveryDisposition::Suspended);
788 }
789
790 let checkpoint_sequence: Option<i64> =
791 sqlx::query_scalar("SELECT MAX(sequence) FROM agent_checkpoints WHERE run_id = ?")
792 .bind(&run.id)
793 .fetch_one(&self.pool)
794 .await?;
795 let Some(checkpoint_sequence) = checkpoint_sequence else {
796 return Ok(RecoveryDisposition::RestartRequired);
797 };
798
799 let unsafe_invocation: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM tool_invocations WHERE run_id = ? AND ((status != 'completed' AND outcome_class IS NOT 'recoverable_error') OR requested_sequence >= ?))")
800 .bind(&run.id)
801 .bind(checkpoint_sequence)
802 .fetch_one(&self.pool)
803 .await?;
804 let artifact_after_checkpoint: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_events WHERE run_id = ? AND event_type = 'artifact.created' AND sequence > ?)")
805 .bind(&run.id)
806 .bind(checkpoint_sequence)
807 .fetch_one(&self.pool)
808 .await?;
809 if unsafe_invocation || artifact_after_checkpoint {
810 return Ok(RecoveryDisposition::ReconciliationRequired);
811 }
812
813 if run.outcome == Some(RunOutcomeKind::LimitExceeded) {
814 return Ok(RecoveryDisposition::RestartRequired);
815 }
816 let delegated: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_run_lineage WHERE (run_id = ? AND parent_run_id IS NOT NULL) OR parent_run_id = ?)")
817 .bind(&run.id)
818 .bind(&run.id)
819 .fetch_one(&self.pool)
820 .await?;
821 if delegated {
822 return Ok(RecoveryDisposition::RestartRequired);
823 }
824 if let Some(timeout) = run.budgets.timeout_seconds {
825 let deadline = i64::try_from(timeout)
826 .ok()
827 .and_then(|seconds| seconds.checked_mul(1000))
828 .and_then(|millis| run.created_at.checked_add(millis));
829 if deadline.is_none_or(|deadline| Utc::now().timestamp_millis() >= deadline) {
830 return Ok(RecoveryDisposition::RestartRequired);
831 }
832 }
833 if run
834 .budgets
835 .max_turns
836 .is_some_and(|limit| run.usage.model_turns >= limit)
837 || run
838 .budgets
839 .max_total_tokens
840 .is_some_and(|limit| run.usage.total_tokens.is_none_or(|total| total >= limit))
841 || run
842 .budgets
843 .max_tool_calls
844 .is_some_and(|limit| run.usage.tool_calls >= limit)
845 {
846 return Ok(RecoveryDisposition::RestartRequired);
847 }
848 Ok(RecoveryDisposition::ResumeEligible)
849 }
850
851 async fn active_budget_chain(
852 tx: &mut Transaction<'_, Sqlite>,
853 id: &str,
854 workspace_id: &str,
855 ) -> Result<Vec<BudgetRow>> {
856 let rows = sqlx::query_as(
857 "WITH RECURSIVE chain(id) AS (SELECT ? UNION ALL SELECT l.parent_run_id FROM agent_run_lineage l JOIN chain c ON l.run_id = c.id WHERE l.parent_run_id IS NOT NULL) SELECT r.id, r.budgets, r.usage FROM chain c JOIN agent_runs r ON r.id = c.id WHERE r.workspace_id = ? AND r.status = 'running'",
858 )
859 .bind(id)
860 .bind(workspace_id)
861 .fetch_all(&mut **tx)
862 .await?;
863 ensure!(
864 !rows.is_empty(),
865 "run not found in workspace or already terminal"
866 );
867 Ok(rows)
868 }
869
870 async fn acquire_run_write(
871 tx: &mut Transaction<'_, Sqlite>,
872 id: &str,
873 workspace_id: &str,
874 ) -> Result<()> {
875 let changed = sqlx::query(
876 "UPDATE agent_runs SET updated_at = updated_at WHERE id = ? AND workspace_id = ? AND lifecycle = 'active'",
877 )
878 .bind(id)
879 .bind(workspace_id)
880 .execute(&mut **tx)
881 .await?
882 .rows_affected();
883 ensure!(changed == 1, "run is not active in workspace");
884 Ok(())
885 }
886
887 async fn save_usage(
888 tx: &mut Transaction<'_, Sqlite>,
889 row: &BudgetRow,
890 usage: &RunUsage,
891 ) -> Result<()> {
892 sqlx::query("UPDATE agent_runs SET usage = ? WHERE id = ?")
893 .bind(sqlx::types::Json(usage))
894 .bind(&row.id)
895 .execute(&mut **tx)
896 .await?;
897 Ok(())
898 }
899
900 pub async fn record_model_requested(
902 &self,
903 id: &str,
904 payload: &Value,
905 ) -> Result<BudgetDecision> {
906 let mut tx = self.pool.begin().await?;
907 Self::acquire_run_write(&mut tx, id, &self.workspace_id).await?;
908 let rows = Self::active_budget_chain(&mut tx, id, &self.workspace_id).await?;
909 if rows.iter().any(|row| {
910 row.budgets.max_total_tokens.is_some() && row.usage.responses_without_usage > 0
911 }) {
912 return Ok(BudgetDecision::Exceeded(LimitReason::TokenUsageUnavailable));
913 }
914 if rows.iter().any(|row| {
915 row.budgets
916 .max_total_tokens
917 .is_some_and(|limit| row.usage.total_tokens.is_some_and(|total| total >= limit))
918 }) {
919 return Ok(BudgetDecision::Exceeded(LimitReason::TotalTokens));
920 }
921 if rows.iter().any(|row| {
922 row.budgets
923 .max_turns
924 .is_some_and(|limit| row.usage.model_turns >= limit)
925 }) {
926 return Ok(BudgetDecision::Exceeded(LimitReason::ModelTurns));
927 }
928 for row in &rows {
929 let mut usage = row.usage.0.clone();
930 usage.model_turns = usage.model_turns.checked_add(1).ok_or(AccountingOverflow)?;
931 Self::save_usage(&mut tx, row, &usage).await?;
932 if row.id != id {
933 Self::refresh_working_state(&mut tx, &row.id).await?;
934 }
935 }
936 self.append_in(&mut tx, id, "model.requested", payload)
937 .await?;
938 tx.commit().await?;
939 Ok(BudgetDecision::Allowed)
940 }
941
942 pub async fn record_model_responded(
944 &self,
945 id: &str,
946 payload: &Value,
947 usage: Option<(u64, u64, u64)>,
948 ) -> Result<BudgetDecision> {
949 let mut tx = self.pool.begin().await?;
950 Self::acquire_run_write(&mut tx, id, &self.workspace_id).await?;
951 let rows = Self::active_budget_chain(&mut tx, id, &self.workspace_id).await?;
952 let mut decision = BudgetDecision::Allowed;
953 for row in &rows {
954 let mut next = row.usage.0.clone();
955 match usage {
956 Some((input, output, total)) => {
957 next.input_tokens = Some(
958 next.input_tokens
959 .unwrap_or(0)
960 .checked_add(input)
961 .ok_or(AccountingOverflow)?,
962 );
963 next.output_tokens = Some(
964 next.output_tokens
965 .unwrap_or(0)
966 .checked_add(output)
967 .ok_or(AccountingOverflow)?,
968 );
969 next.total_tokens = Some(
970 next.total_tokens
971 .unwrap_or(0)
972 .checked_add(total)
973 .ok_or(AccountingOverflow)?,
974 );
975 next.responses_with_usage = next
976 .responses_with_usage
977 .checked_add(1)
978 .ok_or(AccountingOverflow)?;
979 }
980 None => {
981 next.responses_without_usage = next
982 .responses_without_usage
983 .checked_add(1)
984 .ok_or(AccountingOverflow)?;
985 }
986 }
987 next.token_accounting = match (
988 next.responses_with_usage > 0,
989 next.responses_without_usage > 0,
990 ) {
991 (true, false) => TokenAccounting::Complete,
992 (true, true) => TokenAccounting::Partial,
993 _ => TokenAccounting::Unavailable,
994 };
995 if row.budgets.max_total_tokens.is_some() && usage.is_none() {
996 decision = BudgetDecision::Exceeded(LimitReason::TokenUsageUnavailable);
997 } else if row
998 .budgets
999 .max_total_tokens
1000 .is_some_and(|limit| next.total_tokens.is_some_and(|total| total > limit))
1001 {
1002 decision = BudgetDecision::Exceeded(LimitReason::TotalTokens);
1003 }
1004 Self::save_usage(&mut tx, row, &next).await?;
1005 if row.id != id {
1006 Self::refresh_working_state(&mut tx, &row.id).await?;
1007 }
1008 }
1009 self.append_in(&mut tx, id, "model.responded", payload)
1010 .await?;
1011 tx.commit().await?;
1012 Ok(decision)
1013 }
1014
1015 pub async fn check_tool_call_budget(&self, id: &str, calls: usize) -> Result<BudgetDecision> {
1017 let calls = u64::try_from(calls).context("tool-call count overflow")?;
1018 let mut tx = self.pool.begin().await?;
1019 let rows = Self::active_budget_chain(&mut tx, id, &self.workspace_id).await?;
1020 let exceeded = rows.iter().any(|row| {
1021 row.budgets.max_tool_calls.is_some_and(|limit| {
1022 row.usage
1023 .tool_calls
1024 .checked_add(calls)
1025 .is_none_or(|total| total > limit)
1026 })
1027 });
1028 tx.rollback().await?;
1029 Ok(if exceeded {
1030 BudgetDecision::Exceeded(LimitReason::ToolCalls)
1031 } else {
1032 BudgetDecision::Allowed
1033 })
1034 }
1035
1036 pub async fn check_model_turn_budget(&self, id: &str) -> Result<BudgetDecision> {
1037 let mut tx = self.pool.begin().await?;
1038 let rows = Self::active_budget_chain(&mut tx, id, &self.workspace_id).await?;
1039 let decision = if rows.iter().any(|row| {
1040 row.budgets.max_total_tokens.is_some() && row.usage.responses_without_usage > 0
1041 }) {
1042 BudgetDecision::Exceeded(LimitReason::TokenUsageUnavailable)
1043 } else if rows.iter().any(|row| {
1044 row.budgets
1045 .max_total_tokens
1046 .is_some_and(|limit| row.usage.total_tokens.is_some_and(|total| total >= limit))
1047 }) {
1048 BudgetDecision::Exceeded(LimitReason::TotalTokens)
1049 } else if rows.iter().any(|row| {
1050 row.budgets
1051 .max_turns
1052 .is_some_and(|limit| row.usage.model_turns >= limit)
1053 }) {
1054 BudgetDecision::Exceeded(LimitReason::ModelTurns)
1055 } else {
1056 BudgetDecision::Allowed
1057 };
1058 tx.rollback().await?;
1059 Ok(decision)
1060 }
1061
1062 pub async fn append_event(&self, id: &str, event_type: &str, payload: &Value) -> Result<i64> {
1065 ensure!(!event_type.trim().is_empty(), "event type is required");
1066 ensure!(
1067 !event_type.starts_with("run.")
1068 && !event_type.starts_with("checkpoint.")
1069 && !event_type.starts_with("tool.")
1070 && !event_type.starts_with("approval.")
1071 && !event_type.starts_with("artifact.")
1072 && !event_type.starts_with("delegation."),
1073 "reserved event type"
1074 );
1075 let mut tx = self.pool.begin().await?;
1076 let sequence = self.append_in(&mut tx, id, event_type, payload).await?;
1077 tx.commit().await?;
1078 Ok(sequence)
1079 }
1080
1081 pub async fn events(
1083 &self,
1084 id: &str,
1085 after_sequence: i64,
1086 limit: u32,
1087 ) -> Result<Vec<AgentEvent>> {
1088 self.get_run(id)
1089 .await?
1090 .context("run not found in workspace")?;
1091 Ok(sqlx::query_as("SELECT e.* FROM agent_events e JOIN agent_runs r ON r.id = e.run_id WHERE e.run_id = ? AND r.workspace_id = ? AND e.sequence > ? ORDER BY e.sequence LIMIT ?")
1092 .bind(id).bind(&self.workspace_id).bind(after_sequence).bind(limit).fetch_all(&self.pool).await?)
1093 }
1094
1095 pub async fn save_checkpoint(
1097 &self,
1098 id: &str,
1099 schema_version: i64,
1100 turn: i64,
1101 state: &Value,
1102 ) -> Result<i64> {
1103 ensure!(
1104 schema_version > 0 && turn >= 0,
1105 "invalid checkpoint version or turn"
1106 );
1107 let mut tx = self.pool.begin().await?;
1108 let sequence = self
1109 .append_in(
1110 &mut tx,
1111 id,
1112 "checkpoint.created",
1113 &json!({"schema_version": schema_version, "turn": turn}),
1114 )
1115 .await?;
1116 sqlx::query("INSERT INTO agent_checkpoints (run_id, sequence, schema_version, turn, created_at, state) VALUES (?, ?, ?, ?, ?, ?)")
1117 .bind(id).bind(sequence).bind(schema_version).bind(turn).bind(Utc::now().timestamp_millis()).bind(sqlx::types::Json(state))
1118 .execute(&mut *tx).await?;
1119 tx.commit().await?;
1120 Ok(sequence)
1121 }
1122
1123 pub async fn latest_checkpoint(&self, id: &str) -> Result<Option<AgentCheckpoint>> {
1124 self.get_run(id)
1125 .await?
1126 .context("run not found in workspace")?;
1127 Ok(sqlx::query_as("SELECT c.* FROM agent_checkpoints c JOIN agent_runs r ON r.id = c.run_id WHERE c.run_id = ? AND r.workspace_id = ? ORDER BY c.sequence DESC LIMIT 1")
1128 .bind(id).bind(&self.workspace_id).fetch_optional(&self.pool).await?)
1129 }
1130
1131 pub async fn latest_suspension(&self, id: &str) -> Result<Option<RunSuspension>> {
1132 self.get_run(id)
1133 .await?
1134 .context("run not found in workspace")?;
1135 Ok(sqlx::query_as("SELECT s.* FROM agent_run_suspensions s JOIN agent_runs r ON r.id = s.run_id WHERE s.run_id = ? AND r.workspace_id = ? ORDER BY s.suspension_sequence DESC LIMIT 1")
1136 .bind(id)
1137 .bind(&self.workspace_id)
1138 .fetch_optional(&self.pool)
1139 .await?)
1140 }
1141
1142 pub async fn reopen_run(&self, id: &str, expected_sequence: i64) -> Result<AgentRun> {
1146 ensure!(expected_sequence > 0, "invalid expected run sequence");
1147 let mut tx = self.pool.begin().await?;
1148 let previous: Option<String> = sqlx::query_scalar("UPDATE agent_runs SET updated_at = updated_at WHERE id = ? AND workspace_id = ? AND last_sequence = ? AND status IN ('running', 'failed', 'cancelled') AND lifecycle != 'suspended' AND outcome IS NOT 'limit_exceeded' RETURNING status")
1149 .bind(id).bind(&self.workspace_id).bind(expected_sequence).fetch_optional(&mut *tx).await?;
1150 let previous = previous
1151 .context("run not found in workspace, completed, or changed since validation")?;
1152 let delegated: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_run_lineage WHERE (run_id = ? AND parent_run_id IS NOT NULL) OR parent_run_id = ?)")
1153 .bind(id).bind(id).fetch_one(&mut *tx).await?;
1154 ensure!(
1155 !delegated,
1156 "cannot resume delegated child or parent with descendants"
1157 );
1158 let unsafe_tools: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM tool_invocations WHERE run_id = ? AND status != 'completed' AND outcome_class IS NOT 'recoverable_error')")
1159 .bind(id).fetch_one(&mut *tx).await?;
1160 ensure!(
1161 !unsafe_tools,
1162 "cannot resume run with incomplete, failed, or denied tool invocations"
1163 );
1164 sqlx::query("UPDATE agent_runs SET status = 'running', lifecycle = 'active', outcome = NULL, reason_code = NULL, reason_detail = NULL, output = NULL, error = NULL, completed_at = NULL WHERE id = ? AND workspace_id = ?")
1165 .bind(id).bind(&self.workspace_id).execute(&mut *tx).await?;
1166 self.append_in(
1167 &mut tx,
1168 id,
1169 "run.resumed",
1170 &json!({"previous_status": previous, "previous_sequence": expected_sequence}),
1171 )
1172 .await?;
1173 let run = sqlx::query_as("SELECT *, MAX(0, CAST(unixepoch('subsec') * 1000 AS INTEGER) - created_at) AS elapsed_ms, MAX(0, CAST(unixepoch('subsec') * 1000 AS INTEGER) - created_at - suspended_ms) AS execution_elapsed_ms FROM agent_runs WHERE id = ? AND workspace_id = ?")
1174 .bind(id)
1175 .bind(&self.workspace_id)
1176 .fetch_one(&mut *tx)
1177 .await?;
1178 tx.commit().await?;
1179 Ok(run)
1180 }
1181
1182 #[allow(clippy::too_many_arguments)]
1185 pub async fn continue_suspended_run(
1186 &self,
1187 id: &str,
1188 expected_sequence: i64,
1189 suspension_sequence: i64,
1190 input: &str,
1191 checkpoint_schema: i64,
1192 checkpoint_turn: i64,
1193 checkpoint_state: &Value,
1194 ) -> Result<AgentRun> {
1195 ensure!(
1196 !input.trim().is_empty() && input.len() <= 8192,
1197 "continuation input must contain 1–8192 bytes"
1198 );
1199 ensure!(
1200 expected_sequence > 0
1201 && suspension_sequence > 0
1202 && checkpoint_schema > 0
1203 && checkpoint_turn >= 0,
1204 "invalid continuation boundary"
1205 );
1206 ensure!(
1207 serde_json::to_vec(checkpoint_state)?.len() <= 16 * 1024 * 1024,
1208 "checkpoint exceeds 16 MiB"
1209 );
1210 let mut tx = self.pool.begin().await?;
1211 let row: Option<ContinuationRunRow> = sqlx::query_as("UPDATE agent_runs SET updated_at = updated_at WHERE id = ? AND workspace_id = ? AND last_sequence = ? AND lifecycle = 'suspended' AND outcome IN ('blocked', 'needs_user_input') AND completed_at IS NOT NULL RETURNING created_at, completed_at, suspended_ms, budgets, usage")
1212 .bind(id)
1213 .bind(&self.workspace_id)
1214 .bind(expected_sequence)
1215 .fetch_optional(&mut *tx)
1216 .await?;
1217 let row = row.context("run is not the expected current suspension")?;
1218 let now = Utc::now().timestamp_millis();
1219 let interval = now
1220 .checked_sub(row.completed_at)
1221 .filter(|value| *value >= 0)
1222 .context("invalid suspended interval")?;
1223 let suspension: Option<(RunOutcomeKind, String)> = sqlx::query_as("SELECT outcome, reason_code FROM agent_run_suspensions WHERE run_id = ? AND suspension_sequence = ? AND continuation_input IS NULL")
1224 .bind(id)
1225 .bind(suspension_sequence)
1226 .fetch_optional(&mut *tx)
1227 .await?;
1228 let (prior_outcome, reason_code) =
1229 suspension.context("pending suspension is unavailable or already continued")?;
1230 ensure!(
1231 suspension_sequence == expected_sequence,
1232 "suspension is no longer current"
1233 );
1234 sqlx::query("UPDATE agent_run_suspensions SET continuation_input = ?, continued_at = ? WHERE run_id = ? AND suspension_sequence = ? AND continuation_input IS NULL")
1235 .bind(input)
1236 .bind(now)
1237 .bind(id)
1238 .bind(suspension_sequence)
1239 .execute(&mut *tx)
1240 .await?;
1241 let suspended_ms = row
1242 .suspended_ms
1243 .checked_add(interval)
1244 .context("suspended duration overflow")?;
1245 let mut checkpoint_state = checkpoint_state.clone();
1246 if let Some(state) = checkpoint_state.as_object_mut() {
1247 state.insert("suspended_ms".into(), json!(suspended_ms));
1248 if let Some(timeout) = row.budgets.timeout_seconds {
1249 let deadline = row
1250 .created_at
1251 .checked_add(
1252 i64::try_from(timeout)?
1253 .checked_mul(1000)
1254 .context("timeout is too large")?,
1255 )
1256 .and_then(|value| value.checked_add(suspended_ms))
1257 .context("timeout is too large")?;
1258 state.insert("original_deadline".into(), json!(deadline));
1259 }
1260 }
1261 sqlx::query("UPDATE agent_runs SET status = 'running', lifecycle = 'active', outcome = NULL, reason_code = NULL, reason_detail = NULL, output = NULL, error = NULL, completed_at = NULL, suspended_ms = suspended_ms + ? WHERE id = ? AND workspace_id = ?")
1262 .bind(interval)
1263 .bind(id)
1264 .bind(&self.workspace_id)
1265 .execute(&mut *tx)
1266 .await?;
1267 let checkpoint_sequence = self
1268 .append_in(
1269 &mut tx,
1270 id,
1271 "checkpoint.created",
1272 &json!({"schema_version": checkpoint_schema, "turn": checkpoint_turn}),
1273 )
1274 .await?;
1275 sqlx::query("INSERT INTO agent_checkpoints (run_id, sequence, schema_version, turn, created_at, state) VALUES (?, ?, ?, ?, ?, ?)")
1276 .bind(id)
1277 .bind(checkpoint_sequence)
1278 .bind(checkpoint_schema)
1279 .bind(checkpoint_turn)
1280 .bind(now)
1281 .bind(sqlx::types::Json(&checkpoint_state))
1282 .execute(&mut *tx)
1283 .await?;
1284 self.append_in(
1285 &mut tx,
1286 id,
1287 "run.continued",
1288 &json!({"prior_outcome": prior_outcome, "reason_code": reason_code, "suspension_sequence": suspension_sequence, "continuation_bytes": input.len(), "suspended_ms": suspended_ms, "budgets": row.budgets, "usage": row.usage}),
1289 )
1290 .await?;
1291 let run = sqlx::query_as("SELECT *, MAX(0, CAST(unixepoch('subsec') * 1000 AS INTEGER) - created_at) AS elapsed_ms, MAX(0, CAST(unixepoch('subsec') * 1000 AS INTEGER) - created_at - suspended_ms) AS execution_elapsed_ms FROM agent_runs WHERE id = ? AND workspace_id = ?")
1292 .bind(id)
1293 .bind(&self.workspace_id)
1294 .fetch_one(&mut *tx)
1295 .await?;
1296 tx.commit().await?;
1297 Ok(run)
1298 }
1299
1300 pub async fn record_artifact(
1304 &self,
1305 id: &str,
1306 relative_path: &str,
1307 sha256: &str,
1308 size: u64,
1309 ) -> Result<i64> {
1310 ensure!(
1311 !relative_path.is_empty()
1312 && relative_path.len() <= 1024
1313 && !relative_path.contains(['\\', ':'])
1314 && !relative_path.chars().any(char::is_control)
1315 && relative_path
1316 .split('/')
1317 .all(|part| !part.is_empty() && part != "." && part != ".."),
1318 "artifact path must be normalized and relative"
1319 );
1320 ensure!(
1321 sha256.len() == 64
1322 && sha256
1323 .bytes()
1324 .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),
1325 "invalid artifact SHA256"
1326 );
1327 ensure!(size <= MAX_ARTIFACT_BYTES, "artifact exceeds size limit");
1328 let mut tx = self.pool.begin().await?;
1329 let found: Option<String> = sqlx::query_scalar("UPDATE agent_runs SET updated_at = updated_at WHERE id = ? AND workspace_id = ? AND status = 'running' RETURNING id")
1331 .bind(id).bind(&self.workspace_id).fetch_optional(&mut *tx).await?;
1332 ensure!(
1333 found.is_some(),
1334 "run not found in workspace or already terminal"
1335 );
1336 let exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM agent_events WHERE run_id = ? AND event_type = 'artifact.created' AND json_extract(payload, '$.relative_path') = ?)")
1337 .bind(id).bind(relative_path).fetch_one(&mut *tx).await?;
1338 ensure!(!exists, "artifact path already recorded");
1339 let sequence = self
1340 .append_in(
1341 &mut tx,
1342 id,
1343 "artifact.created",
1344 &json!({"relative_path": relative_path, "sha256": sha256, "size": size}),
1345 )
1346 .await?;
1347 tx.commit().await?;
1348 Ok(sequence)
1349 }
1350
1351 pub async fn artifacts(&self, id: &str) -> Result<Vec<ArtifactMetadata>> {
1352 self.get_run(id)
1353 .await?
1354 .context("run not found in workspace")?;
1355 let events: Vec<AgentEvent> = sqlx::query_as("SELECT e.* FROM agent_events e JOIN agent_runs r ON r.id = e.run_id WHERE e.run_id = ? AND r.workspace_id = ? AND e.event_type = 'artifact.created' ORDER BY e.sequence")
1356 .bind(id).bind(&self.workspace_id).fetch_all(&self.pool).await?;
1357 events
1358 .into_iter()
1359 .map(|event| {
1360 let mut payload = event.payload.0;
1361 payload["sequence"] = json!(event.sequence);
1362 serde_json::from_value(payload).context("invalid artifact metadata")
1363 })
1364 .collect()
1365 }
1366
1367 pub async fn finish_run(&self, id: &str, outcome: RunOutcome) -> Result<()> {
1369 self.finish_run_with_pending_suspension(id, outcome, None)
1370 .await
1371 }
1372
1373 pub(crate) async fn finish_run_with_pending_suspension(
1374 &self,
1375 id: &str,
1376 outcome: RunOutcome,
1377 pending_assistant: Option<Value>,
1378 ) -> Result<()> {
1379 let pending = match (&outcome, pending_assistant) {
1380 (RunOutcome::Blocked { message, .. }, assistant) => Some((message.clone(), assistant)),
1381 (RunOutcome::NeedsUserInput { question, .. }, assistant) => {
1382 Some((question.clone(), assistant))
1383 }
1384 _ => None,
1385 };
1386 let stopped = stopped(outcome)?;
1387 let mut tx = self.pool.begin().await?;
1388 let found: Option<String> = sqlx::query_scalar("UPDATE agent_runs SET updated_at = updated_at WHERE id = ? AND workspace_id = ? AND status = 'running' RETURNING id")
1391 .bind(id).bind(&self.workspace_id).fetch_optional(&mut *tx).await?;
1392 ensure!(
1393 found.is_some(),
1394 "run not found in workspace or already terminal"
1395 );
1396 let descendants: Vec<String> = sqlx::query_scalar("WITH RECURSIVE descendants(id) AS (SELECT run_id FROM agent_run_lineage WHERE parent_run_id = ? UNION ALL SELECT l.run_id FROM agent_run_lineage l JOIN descendants d ON l.parent_run_id = d.id) SELECT r.id FROM descendants d JOIN agent_runs r ON r.id = d.id WHERE r.workspace_id = ? AND r.status = 'running'")
1397 .bind(id).bind(&self.workspace_id).fetch_all(&mut *tx).await?;
1398 ensure!(
1399 stopped.outcome != RunOutcomeKind::Completed || descendants.is_empty(),
1400 "cannot complete run with active descendants"
1401 );
1402 for child in descendants {
1403 let child_outcome = if stopped.outcome == RunOutcomeKind::Cancelled {
1404 StoppedRun {
1405 lifecycle: RunLifecycle::Terminal,
1406 outcome: RunOutcomeKind::Cancelled,
1407 reason_code: "ancestor_cancelled".into(),
1408 reason_detail: json!({}),
1409 status: "cancelled",
1410 output: None,
1411 error: None,
1412 }
1413 } else {
1414 StoppedRun {
1415 lifecycle: RunLifecycle::Terminal,
1416 outcome: RunOutcomeKind::Failed,
1417 reason_code: "tool_error".into(),
1418 reason_detail: json!({}),
1419 status: "failed",
1420 output: None,
1421 error: Some("ancestor run interrupted".into()),
1422 }
1423 };
1424 self.finish_in(&mut tx, &child, &child_outcome).await?;
1425 }
1426 let suspension_sequence = self.finish_in(&mut tx, id, &stopped).await?;
1427 if let Some((prompt, assistant)) = pending {
1428 sqlx::query("INSERT INTO agent_run_suspensions (run_id, suspension_sequence, outcome, reason_code, prompt, assistant_message) VALUES (?, ?, ?, ?, ?, ?)")
1429 .bind(id)
1430 .bind(suspension_sequence)
1431 .bind(stopped.outcome)
1432 .bind(&stopped.reason_code)
1433 .bind(prompt)
1434 .bind(assistant.map(sqlx::types::Json))
1435 .execute(&mut *tx)
1436 .await?;
1437 }
1438 tx.commit().await?;
1439 Ok(())
1440 }
1441
1442 async fn finish_in(
1443 &self,
1444 tx: &mut Transaction<'_, Sqlite>,
1445 id: &str,
1446 stopped: &StoppedRun,
1447 ) -> Result<i64> {
1448 let state: (sqlx::types::Json<RunBudgets>, sqlx::types::Json<RunUsage>) =
1449 sqlx::query_as("SELECT budgets, usage FROM agent_runs WHERE id = ?")
1450 .bind(id)
1451 .fetch_one(&mut **tx)
1452 .await?;
1453 let unfinished: Vec<String> = sqlx::query_scalar("SELECT call_id FROM tool_invocations WHERE run_id = ? AND status IN ('requested', 'started') ORDER BY requested_sequence")
1454 .bind(id).fetch_all(&mut **tx).await?;
1455 ensure!(
1456 stopped.status != "completed" || unfinished.is_empty(),
1457 "cannot complete run with unfinished tools"
1458 );
1459 for call_id in unfinished {
1460 let invocation_outcome = if stopped.outcome == RunOutcomeKind::Cancelled {
1461 "cancelled"
1462 } else {
1463 "terminal_error"
1464 };
1465 self.deny_pending_approval(tx, id, &call_id, "run interrupted")
1466 .await?;
1467 self.append_in(
1468 tx,
1469 id,
1470 "tool.failed",
1471 &json!({"call_id": call_id, "outcome": invocation_outcome, "error": "run interrupted", "message": "run interrupted"}),
1472 )
1473 .await?;
1474 sqlx::query("UPDATE tool_invocations SET status = 'failed', error = 'run interrupted', outcome_class = ?, completed_at = ? WHERE run_id = ? AND call_id = ?")
1475 .bind(invocation_outcome).bind(Utc::now().timestamp_millis()).bind(id).bind(call_id).execute(&mut **tx).await?;
1476 }
1477 let sequence = self.append_in(
1478 tx,
1479 id,
1480 &format!("run.{}", reason_code_for_event(stopped.outcome)),
1481 &json!({"lifecycle": stopped.lifecycle, "outcome": stopped.outcome, "reason_code": stopped.reason_code, "reason_detail": stopped.reason_detail, "usage": state.1, "budgets": state.0, "output": stopped.output, "error": stopped.error}),
1482 )
1483 .await?;
1484 sqlx::query("UPDATE agent_runs SET status = ?, lifecycle = ?, outcome = ?, reason_code = ?, reason_detail = ?, output = ?, error = ?, completed_at = updated_at WHERE id = ? AND workspace_id = ?")
1485 .bind(stopped.status).bind(stopped.lifecycle).bind(stopped.outcome).bind(&stopped.reason_code).bind(sqlx::types::Json(&stopped.reason_detail)).bind(&stopped.output).bind(&stopped.error).bind(id).bind(&self.workspace_id).execute(&mut **tx).await?;
1486 Ok(sequence)
1487 }
1488
1489 pub async fn request_tool(
1490 &self,
1491 id: &str,
1492 call_id: &str,
1493 name: &str,
1494 arguments: &Value,
1495 ) -> Result<()> {
1496 ensure!(
1497 !call_id.is_empty() && !name.is_empty() && arguments.is_object(),
1498 "invalid tool invocation"
1499 );
1500 let mut tx = self.pool.begin().await?;
1501 let sequence = self
1502 .append_in(
1503 &mut tx,
1504 id,
1505 "tool.requested",
1506 &json!({"call_id": call_id, "tool": name}),
1507 )
1508 .await?;
1509 sqlx::query("INSERT INTO tool_invocations (run_id, call_id, requested_sequence, tool_name, arguments, status) VALUES (?, ?, ?, ?, ?, 'requested')")
1510 .bind(id).bind(call_id).bind(sequence).bind(name).bind(sqlx::types::Json(arguments)).execute(&mut *tx).await?;
1511 tx.commit().await?;
1512 Ok(())
1513 }
1514
1515 pub async fn request_approval(
1519 &self,
1520 id: &str,
1521 call_id: &str,
1522 capabilities: &[Capability],
1523 ) -> Result<()> {
1524 ensure!(
1525 !capabilities.is_empty(),
1526 "approval capabilities are required"
1527 );
1528 let unique: std::collections::HashSet<_> = capabilities.iter().collect();
1529 ensure!(
1530 unique.len() == capabilities.len(),
1531 "duplicate approval capability"
1532 );
1533 let mut tx = self.pool.begin().await?;
1534 self.lock_requested_tool(&mut tx, id, call_id).await?;
1535 ensure!(
1536 Self::approval_state(&mut tx, id, call_id).await?.is_none(),
1537 "approval already requested"
1538 );
1539 self.append_in(
1540 &mut tx,
1541 id,
1542 "approval.requested",
1543 &json!({"call_id": call_id, "capabilities": capabilities}),
1544 )
1545 .await?;
1546 tx.commit().await?;
1547 Ok(())
1548 }
1549
1550 pub async fn decide_approval(&self, id: &str, call_id: &str, granted: bool) -> Result<()> {
1551 let mut tx = self.pool.begin().await?;
1552 self.lock_requested_tool(&mut tx, id, call_id).await?;
1553 ensure!(
1554 Self::approval_state(&mut tx, id, call_id).await?.as_deref()
1555 == Some("approval.requested"),
1556 "approval is not pending"
1557 );
1558 let event_type = if granted {
1559 "approval.granted"
1560 } else {
1561 "approval.denied"
1562 };
1563 self.append_in(&mut tx, id, event_type, &json!({"call_id": call_id}))
1564 .await?;
1565 tx.commit().await?;
1566 Ok(())
1567 }
1568
1569 async fn lock_requested_tool(
1570 &self,
1571 tx: &mut Transaction<'_, Sqlite>,
1572 id: &str,
1573 call_id: &str,
1574 ) -> Result<()> {
1575 let found: Option<String> = sqlx::query_scalar("UPDATE agent_runs SET updated_at = updated_at WHERE id = ? AND workspace_id = ? AND status = 'running' RETURNING id")
1578 .bind(id).bind(&self.workspace_id).fetch_optional(&mut **tx).await?;
1579 ensure!(
1580 found.is_some(),
1581 "run not found in workspace or already terminal"
1582 );
1583 let pending: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM tool_invocations WHERE run_id = ? AND call_id = ? AND status = 'requested')")
1584 .bind(id).bind(call_id).fetch_one(&mut **tx).await?;
1585 ensure!(pending, "tool invocation is not pending");
1586 Ok(())
1587 }
1588
1589 async fn approval_state(
1590 tx: &mut Transaction<'_, Sqlite>,
1591 id: &str,
1592 call_id: &str,
1593 ) -> Result<Option<String>> {
1594 Ok(sqlx::query_scalar("SELECT event_type FROM agent_events WHERE run_id = ? AND event_type IN ('approval.requested', 'approval.granted', 'approval.denied') AND json_extract(payload, '$.call_id') = ? ORDER BY sequence DESC LIMIT 1")
1595 .bind(id).bind(call_id).fetch_optional(&mut **tx).await?)
1596 }
1597
1598 async fn deny_pending_approval(
1599 &self,
1600 tx: &mut Transaction<'_, Sqlite>,
1601 id: &str,
1602 call_id: &str,
1603 reason: &str,
1604 ) -> Result<()> {
1605 if Self::approval_state(tx, id, call_id).await?.as_deref() == Some("approval.requested") {
1606 self.append_in(
1607 tx,
1608 id,
1609 "approval.denied",
1610 &json!({"call_id": call_id, "reason": reason}),
1611 )
1612 .await?;
1613 }
1614 Ok(())
1615 }
1616
1617 pub async fn start_tool(&self, id: &str, call_id: &str) -> Result<()> {
1618 let mut tx = self.pool.begin().await?;
1619 Self::acquire_run_write(&mut tx, id, &self.workspace_id).await?;
1620 let rows = Self::active_budget_chain(&mut tx, id, &self.workspace_id).await?;
1621 ensure!(
1622 !rows.iter().any(|row| row
1623 .budgets
1624 .max_tool_calls
1625 .is_some_and(|limit| row.usage.tool_calls >= limit)),
1626 "tool-call budget exhausted"
1627 );
1628 let approval = Self::approval_state(&mut tx, id, call_id).await?;
1629 ensure!(
1630 approval.is_none() || approval.as_deref() == Some("approval.granted"),
1631 "tool approval is pending or denied"
1632 );
1633 let changed = sqlx::query("UPDATE tool_invocations SET status = 'started', started_at = ? WHERE run_id = ? AND call_id = ? AND status = 'requested'")
1634 .bind(Utc::now().timestamp_millis()).bind(id).bind(call_id).execute(&mut *tx).await?.rows_affected();
1635 ensure!(changed == 1, "tool invocation is not pending");
1636 for row in &rows {
1637 let mut usage = row.usage.0.clone();
1638 usage.tool_calls = usage.tool_calls.checked_add(1).ok_or(AccountingOverflow)?;
1639 Self::save_usage(&mut tx, row, &usage).await?;
1640 if row.id != id {
1641 Self::refresh_working_state(&mut tx, &row.id).await?;
1642 }
1643 }
1644 self.append_in(&mut tx, id, "tool.started", &json!({"call_id": call_id}))
1645 .await?;
1646 tx.commit().await?;
1647 Ok(())
1648 }
1649
1650 pub async fn finish_tool(&self, id: &str, call_id: &str, outcome: ToolOutcome) -> Result<()> {
1651 self.finish_tool_with_count(id, call_id, outcome)
1652 .await
1653 .map(|_| ())
1654 }
1655
1656 pub(crate) async fn finish_tool_with_count(
1657 &self,
1658 id: &str,
1659 call_id: &str,
1660 outcome: ToolOutcome,
1661 ) -> Result<Option<u32>> {
1662 let (
1663 status,
1664 event,
1665 expected,
1666 result,
1667 result_content,
1668 error,
1669 outcome_class,
1670 error_code,
1671 repetition_key,
1672 ) = match outcome {
1673 ToolOutcome::Completed(result) => (
1674 "completed",
1675 "tool.completed",
1676 "started",
1677 Some(sqlx::types::Json(result)),
1678 None,
1679 None,
1680 "completed",
1681 None,
1682 None,
1683 ),
1684 ToolOutcome::Failed(error) => (
1685 "failed",
1686 "tool.failed",
1687 "started",
1688 None,
1689 None,
1690 Some(error),
1691 "terminal_error",
1692 None,
1693 None,
1694 ),
1695 ToolOutcome::Denied(error) => (
1696 "denied",
1697 "tool.denied",
1698 "requested",
1699 None,
1700 None,
1701 Some(error),
1702 "denied",
1703 None,
1704 None,
1705 ),
1706 ToolOutcome::Recoverable {
1707 code,
1708 message,
1709 observation,
1710 observation_content,
1711 repetition_key,
1712 } => (
1713 "failed",
1714 "tool.recoverable_error",
1715 "started",
1716 Some(sqlx::types::Json(observation)),
1717 Some(observation_content),
1718 Some(message),
1719 "recoverable_error",
1720 Some(code),
1721 Some(repetition_key),
1722 ),
1723 ToolOutcome::Terminal { code, message } => (
1724 "failed",
1725 "tool.failed",
1726 "started",
1727 None,
1728 None,
1729 Some(message),
1730 "terminal_error",
1731 Some(code),
1732 None,
1733 ),
1734 ToolOutcome::UncertainSideEffect { code, message } => (
1735 "failed",
1736 "tool.failed",
1737 "started",
1738 None,
1739 None,
1740 Some(message),
1741 "uncertain_side_effect",
1742 Some(code),
1743 None,
1744 ),
1745 };
1746 let mut tx = self.pool.begin().await?;
1747 let consecutive_count = if outcome_class == "recoverable_error" {
1748 let key = repetition_key
1749 .as_deref()
1750 .context("recoverable repetition key missing")?;
1751 let previous: Option<(Option<String>, Option<String>, Option<i64>)> = sqlx::query_as(
1752 "SELECT outcome_class, repetition_key, consecutive_count FROM tool_invocations WHERE run_id = ? AND requested_sequence < (SELECT requested_sequence FROM tool_invocations WHERE run_id = ? AND call_id = ?) ORDER BY requested_sequence DESC LIMIT 1",
1753 )
1754 .bind(id)
1755 .bind(id)
1756 .bind(call_id)
1757 .fetch_optional(&mut *tx)
1758 .await?;
1759 let count = match previous {
1760 Some((Some(class), Some(previous_key), Some(count)))
1761 if class == "recoverable_error" && previous_key == key =>
1762 {
1763 count
1764 .checked_add(1)
1765 .context("recoverable repetition count overflow")?
1766 }
1767 _ => 1,
1768 };
1769 Some(count)
1770 } else {
1771 None
1772 };
1773 self.append_in(
1774 &mut tx,
1775 id,
1776 event,
1777 &json!({
1778 "call_id": call_id,
1779 "outcome": outcome_class,
1780 "code": error_code,
1781 "error": error,
1782 "message": error,
1783 "repetition_key": repetition_key,
1784 "consecutive_count": consecutive_count,
1785 }),
1786 )
1787 .await?;
1788 let changed = sqlx::query("UPDATE tool_invocations SET status = ?, result = ?, result_content = ?, error = ?, outcome_class = ?, error_code = ?, repetition_key = ?, consecutive_count = ?, completed_at = ? WHERE run_id = ? AND call_id = ? AND status = ?")
1789 .bind(status).bind(result).bind(result_content).bind(error).bind(outcome_class).bind(error_code).bind(repetition_key).bind(consecutive_count).bind(Utc::now().timestamp_millis()).bind(id).bind(call_id).bind(expected).execute(&mut *tx).await?.rows_affected();
1790 ensure!(changed == 1, "invalid tool invocation transition");
1791 self.deny_pending_approval(&mut tx, id, call_id, "tool denied")
1792 .await?;
1793 tx.commit().await?;
1794 consecutive_count
1795 .map(u32::try_from)
1796 .transpose()
1797 .context("recoverable repetition count is invalid")
1798 }
1799
1800 pub async fn tool_invocations(&self, id: &str) -> Result<Vec<ToolInvocation>> {
1801 self.get_run(id)
1802 .await?
1803 .context("run not found in workspace")?;
1804 Ok(sqlx::query_as("SELECT t.* FROM tool_invocations t JOIN agent_runs r ON r.id = t.run_id WHERE t.run_id = ? AND r.workspace_id = ? ORDER BY t.requested_sequence")
1805 .bind(id).bind(&self.workspace_id).fetch_all(&self.pool).await?)
1806 }
1807
1808 async fn project_working_state(
1809 tx: &mut Transaction<'_, Sqlite>,
1810 id: &str,
1811 event_type: &str,
1812 payload: &Value,
1813 ) -> Result<()> {
1814 let (lifecycle, outcome, reason_code, usage, cursor): (RunLifecycle, Option<RunOutcomeKind>, Option<String>, sqlx::types::Json<RunUsage>, i64) =
1815 sqlx::query_as("SELECT lifecycle, outcome, reason_code, usage, last_sequence FROM agent_runs WHERE id = ?")
1816 .bind(id).fetch_one(&mut **tx).await?;
1817 let projected_run = if event_type.starts_with("run.") && payload.get("lifecycle").is_some()
1818 {
1819 WorkingStateRun {
1820 lifecycle: serde_json::from_value(payload["lifecycle"].clone())?,
1821 outcome: serde_json::from_value(payload["outcome"].clone())?,
1822 reason_code: serde_json::from_value(payload["reason_code"].clone())?,
1823 }
1824 } else {
1825 WorkingStateRun {
1826 lifecycle,
1827 outcome,
1828 reason_code,
1829 }
1830 };
1831 let total: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM agent_events WHERE run_id = ? AND event_type = 'artifact.created' AND sequence <= ?")
1832 .bind(id).bind(cursor).fetch_one(&mut **tx).await?;
1833 let rows: Vec<(i64, sqlx::types::Json<Value>)> = sqlx::query_as("SELECT sequence, payload FROM agent_events WHERE run_id = ? AND event_type = 'artifact.created' AND sequence <= ? ORDER BY sequence DESC LIMIT 128")
1834 .bind(id).bind(cursor).fetch_all(&mut **tx).await?;
1835 let mut items: Vec<ArtifactMetadata> = rows
1836 .into_iter()
1837 .rev()
1838 .map(|(sequence, mut value)| {
1839 value.0["sequence"] = json!(sequence);
1840 serde_json::from_value(value.0).context("invalid artifact metadata")
1841 })
1842 .collect::<Result<_>>()?;
1843 items.sort_by(|a, b| {
1844 a.sequence
1845 .cmp(&b.sequence)
1846 .then_with(|| a.relative_path.cmp(&b.relative_path))
1847 });
1848 let retained = i64::try_from(items.len())?;
1849 let snapshot = WorkingStateSnapshot {
1850 objective: WorkingStateObjective {
1851 kind: "run_input".into(),
1852 run_id: id.into(),
1853 },
1854 run: projected_run,
1855 usage: usage.0,
1856 latest_event: WorkingStateEventReference {
1857 sequence: cursor,
1858 event_type: event_type.into(),
1859 },
1860 artifacts: WorkingStateArtifacts {
1861 items,
1862 total,
1863 omitted: total - retained,
1864 },
1865 };
1866 let snapshot_json = serde_json::to_value(&snapshot)?;
1867 ensure!(
1868 serde_json::to_vec(&snapshot_json)?.len() <= MAX_WORKING_STATE_BYTES,
1869 "working state exceeds 256 KiB"
1870 );
1871 let previous_revision: Option<i64> =
1872 sqlx::query_scalar("SELECT revision FROM agent_run_working_state WHERE run_id = ?")
1873 .bind(id)
1874 .fetch_optional(&mut **tx)
1875 .await?;
1876 let revision = previous_revision.map_or(Ok(1), |value| {
1877 value
1878 .checked_add(1)
1879 .context("working state revision overflow")
1880 })?;
1881 sqlx::query("INSERT INTO agent_run_working_state (run_id, projection_version, revision, event_cursor, updated_at, snapshot) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(run_id) DO UPDATE SET projection_version = excluded.projection_version, revision = excluded.revision, event_cursor = excluded.event_cursor, updated_at = excluded.updated_at, snapshot = excluded.snapshot")
1882 .bind(id).bind(WORKING_STATE_VERSION).bind(revision).bind(cursor).bind(Utc::now().timestamp_millis()).bind(sqlx::types::Json(snapshot_json)).execute(&mut **tx).await?;
1883 Ok(())
1884 }
1885
1886 async fn refresh_working_state(tx: &mut Transaction<'_, Sqlite>, id: &str) -> Result<()> {
1887 let (event_type, payload): (String, sqlx::types::Json<Value>) = sqlx::query_as(
1888 "SELECT event_type, payload FROM agent_events WHERE run_id = ? ORDER BY sequence DESC LIMIT 1",
1889 ).bind(id).fetch_one(&mut **tx).await?;
1890 Self::project_working_state(tx, id, &event_type, &payload.0).await
1891 }
1892
1893 pub async fn working_state(&self, id: &str) -> Result<WorkingStateInspection> {
1894 let run = self
1895 .get_run_raw(id)
1896 .await?
1897 .context("run not found in workspace")?;
1898 let row: Option<WorkingStateRow> = sqlx::query_as("SELECT s.projection_version, s.revision, s.event_cursor, s.updated_at, s.snapshot FROM agent_run_working_state s JOIN agent_runs r ON r.id = s.run_id WHERE s.run_id = ? AND r.workspace_id = ?")
1899 .bind(id).bind(&self.workspace_id).fetch_optional(&self.pool).await?;
1900 let Some(row) = row else {
1901 return Ok(WorkingStateInspection {
1902 status: WorkingStateStatus::Unavailable,
1903 projection_version: None,
1904 revision: None,
1905 event_cursor: None,
1906 updated_at: None,
1907 snapshot: None,
1908 });
1909 };
1910 let status = if row.projection_version != WORKING_STATE_VERSION {
1911 WorkingStateStatus::Unsupported
1912 } else if row.revision <= 0 || row.event_cursor < 0 || row.event_cursor > run.last_sequence
1913 {
1914 WorkingStateStatus::Unavailable
1915 } else if row.event_cursor < run.last_sequence {
1916 WorkingStateStatus::Stale
1917 } else {
1918 WorkingStateStatus::Current
1919 };
1920 let snapshot: Option<WorkingStateSnapshot> = if matches!(
1921 status,
1922 WorkingStateStatus::Current | WorkingStateStatus::Stale
1923 ) {
1924 serde_json::from_value(row.snapshot.0).ok()
1925 } else {
1926 None
1927 };
1928 let canonical_event_type: Option<String> = if matches!(
1929 status,
1930 WorkingStateStatus::Current | WorkingStateStatus::Stale
1931 ) {
1932 sqlx::query_scalar(
1933 "SELECT event_type FROM agent_events WHERE run_id = ? AND sequence = ?",
1934 )
1935 .bind(id)
1936 .bind(row.event_cursor)
1937 .fetch_optional(&self.pool)
1938 .await?
1939 } else {
1940 None
1941 };
1942 let canonical_artifacts: Option<WorkingStateArtifacts> = if matches!(
1943 status,
1944 WorkingStateStatus::Current | WorkingStateStatus::Stale
1945 ) {
1946 let total: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM agent_events WHERE run_id = ? AND event_type = 'artifact.created' AND sequence <= ?")
1947 .bind(id).bind(row.event_cursor).fetch_one(&self.pool).await?;
1948 let rows: Vec<(i64, sqlx::types::Json<Value>)> = sqlx::query_as("SELECT sequence, payload FROM agent_events WHERE run_id = ? AND event_type = 'artifact.created' AND sequence <= ? ORDER BY sequence DESC LIMIT 128")
1949 .bind(id).bind(row.event_cursor).fetch_all(&self.pool).await?;
1950 let mut items = rows
1951 .into_iter()
1952 .map(|(sequence, mut value)| {
1953 value.0["sequence"] = json!(sequence);
1954 serde_json::from_value(value.0)
1955 })
1956 .collect::<std::result::Result<Vec<ArtifactMetadata>, _>>()
1957 .ok();
1958 items.as_mut().map(|items| {
1959 items.sort_by(|a, b| {
1960 a.sequence
1961 .cmp(&b.sequence)
1962 .then_with(|| a.relative_path.cmp(&b.relative_path))
1963 });
1964 WorkingStateArtifacts {
1965 omitted: total - i64::try_from(items.len()).unwrap_or(i64::MAX),
1966 total,
1967 items: std::mem::take(items),
1968 }
1969 })
1970 } else {
1971 None
1972 };
1973 let structurally_valid = snapshot.as_ref().is_some_and(|snapshot| {
1974 snapshot.objective.kind == "run_input"
1975 && snapshot.objective.run_id == id
1976 && snapshot.latest_event.sequence == row.event_cursor
1977 && canonical_event_type.as_deref()
1978 == Some(snapshot.latest_event.event_type.as_str())
1979 && snapshot.artifacts.total >= 0
1980 && snapshot.artifacts.omitted >= 0
1981 && snapshot.artifacts.items.len() <= 128
1982 && canonical_artifacts.as_ref() == Some(&snapshot.artifacts)
1983 && snapshot
1984 .artifacts
1985 .items
1986 .iter()
1987 .all(|item| item.sequence <= row.event_cursor)
1988 && snapshot.artifacts.total
1989 == snapshot.artifacts.omitted
1990 + i64::try_from(snapshot.artifacts.items.len()).unwrap_or(i64::MAX)
1991 && snapshot
1992 .artifacts
1993 .items
1994 .windows(2)
1995 .all(|pair| pair[0].sequence <= pair[1].sequence)
1996 });
1997 let current_values_match = snapshot.as_ref().is_some_and(|snapshot| {
1998 row.event_cursor != run.last_sequence
1999 || (snapshot.run.lifecycle == run.lifecycle
2000 && snapshot.run.outcome == run.outcome
2001 && snapshot.run.reason_code == run.reason_code
2002 && snapshot.usage == run.usage.0)
2003 });
2004 let status = if (!structurally_valid || !current_values_match)
2005 && matches!(
2006 status,
2007 WorkingStateStatus::Current | WorkingStateStatus::Stale
2008 ) {
2009 WorkingStateStatus::Unavailable
2010 } else {
2011 status
2012 };
2013 let snapshot = if matches!(
2014 status,
2015 WorkingStateStatus::Current | WorkingStateStatus::Stale
2016 ) {
2017 snapshot
2018 } else {
2019 None
2020 };
2021 Ok(WorkingStateInspection {
2022 status,
2023 projection_version: Some(row.projection_version),
2024 revision: Some(row.revision),
2025 event_cursor: Some(row.event_cursor),
2026 updated_at: Some(row.updated_at),
2027 snapshot,
2028 })
2029 }
2030
2031 pub async fn rebuild_working_state(&self, id: &str) -> Result<WorkingStateInspection> {
2032 let mut tx = self.pool.begin().await?;
2033 let found: Option<String> = sqlx::query_scalar(
2034 "UPDATE agent_runs SET updated_at = updated_at WHERE id = ? AND workspace_id = ? RETURNING id",
2035 )
2036 .bind(id)
2037 .bind(&self.workspace_id)
2038 .fetch_optional(&mut *tx)
2039 .await?;
2040 ensure!(found.is_some(), "run not found in workspace");
2041 Self::refresh_working_state(&mut tx, id).await?;
2042 tx.commit().await?;
2043 self.working_state(id).await
2044 }
2045
2046 async fn append_in(
2047 &self,
2048 tx: &mut Transaction<'_, Sqlite>,
2049 id: &str,
2050 event_type: &str,
2051 payload: &Value,
2052 ) -> Result<i64> {
2053 let now = Utc::now().timestamp_millis();
2054 let sequence: i64 = sqlx::query_scalar("UPDATE agent_runs SET last_sequence = last_sequence + 1, updated_at = ? WHERE id = ? AND workspace_id = ? AND status = 'running' RETURNING last_sequence")
2055 .bind(now).bind(id).bind(&self.workspace_id).fetch_optional(&mut **tx).await?
2056 .context("run not found in workspace or already terminal")?;
2057 sqlx::query("INSERT INTO agent_events (run_id, sequence, timestamp, event_type, payload) VALUES (?, ?, ?, ?, ?)")
2058 .bind(id).bind(sequence).bind(now).bind(event_type).bind(sqlx::types::Json(payload)).execute(&mut **tx).await?;
2059 Self::project_working_state(tx, id, event_type, payload).await?;
2060 Ok(sequence)
2061 }
2062}