feat(workflows): M4 — nested sub-workflows (start_child, parent-park, depth ceiling)

A `workflow`-kind step now starts a nested sub-workflow run instead of a
function. The orchestrator's step processing is restructured into prepare →
dispatch (`Prep`): after the shared context/`when`/input resolution, a
function step runs through the executor as before, while a workflow step:

- checks the nesting depth ceiling (`PICLOUD_WORKFLOW_MAX_DEPTH`; the child
  would run at parent depth + 1) — over-deep → the step fails, not infinite;
- resolves the child workflow by name in the run's app scope;
- `start_child_and_park`: in one token-gated tx, seeds a child run (depth + 1,
  `parent_run_id`/`parent_step_id` linkage, correlated under the same
  `root_execution_id`) and PARKS the parent step (`running`, claim_token
  cleared, `child_run_id` set). A parked step is never re-claimed (only `ready`
  steps are) nor reclaimed (only *leased* running steps). The token-gated park
  runs before the child insert, so a stale claim writes nothing (no orphan).

Each tick first runs `resume_finished_children`: a parent step parked on a
now-terminal child is resolved (child output → parent step output, or child
error → parent step failed) and the parent run advanced — idempotent via a
`status='running'` gate. The child's own steps are claimed by the same global
scan, so nesting is just more runs. A failed sub-workflow honors the parent
step's `on_error`.

Shared plumbing extracted for reuse: `advance_run_tx` (out of
`complete_step_and_advance`) and `seed_run_tx` (out of `start_run`).

Apply-time soft-warn (`workflow_nesting_warnings`, wired into `plan_warnings`):
a workflow that nests into itself — directly or via a mutual cycle within the
bundle — is flagged at plan (bounded by the depth ceiling, but almost always a
bug). Not an error.

Tests: nested output-flows-to-parent + depth-ceiling-fails-the-run (DB-gated,
end-to-end), self/mutual-cycle warning (pure unit). fmt + clippy -D warnings
clean, 449 manager-core lib tests, 13 orchestrator DB tests, 26 CLI journeys
(workflows/apply/plan/prune) green, binary boots.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-12 17:30:16 +02:00
parent 41a21c0551
commit 74d5874384
4 changed files with 674 additions and 109 deletions

View File

@@ -3616,6 +3616,7 @@ impl ApplyService {
warnings.extend(unreachable_endpoint_warnings(bundle)); warnings.extend(unreachable_endpoint_warnings(bundle));
} }
warnings.extend(self.dangling_suppress_warnings(owner, bundle).await?); warnings.extend(self.dangling_suppress_warnings(owner, bundle).await?);
warnings.extend(workflow_nesting_warnings(bundle));
Ok(warnings) Ok(warnings)
} }
@@ -5280,6 +5281,64 @@ fn disabled_target_warnings(bundle: &Bundle) -> Vec<String> {
out out
} }
/// M4 soft-warn: a `workflow`-kind step that (directly or transitively, within
/// this bundle) nests back into its own workflow forms a cycle. This is not an
/// error — the runtime depth ceiling (`PICLOUD_WORKFLOW_MAX_DEPTH`) bounds it —
/// but it's almost always a mistake, so we surface it at plan time. Only edges
/// to workflows present in the same bundle are considered (a reference to an
/// external workflow can't be resolved here).
fn workflow_nesting_warnings(bundle: &Bundle) -> Vec<String> {
use std::collections::{BTreeSet, HashMap, HashSet};
if bundle.workflows.is_empty() {
return Vec::new();
}
let names: HashSet<String> = bundle
.workflows
.iter()
.map(|w| w.name.to_lowercase())
.collect();
// from → the set of in-bundle workflows it nests into.
let mut edges: HashMap<String, BTreeSet<String>> = HashMap::new();
for w in &bundle.workflows {
let from = w.name.to_lowercase();
for s in &w.definition.steps {
if let Some(t) = s.workflow.as_deref() {
let t = t.to_lowercase();
if names.contains(&t) {
edges.entry(from.clone()).or_default().insert(t);
}
}
}
}
let mut out = Vec::new();
for w in &bundle.workflows {
let start = w.name.to_lowercase();
// Reachability from `start`: if we can get back to `start`, it's cyclic.
let mut seen: HashSet<String> = HashSet::new();
let mut stack: Vec<String> = edges.get(&start).into_iter().flatten().cloned().collect();
let mut cyclic = false;
while let Some(n) = stack.pop() {
if n == start {
cyclic = true;
break;
}
if seen.insert(n.clone()) {
if let Some(next) = edges.get(&n) {
stack.extend(next.iter().cloned());
}
}
}
if cyclic {
out.push(format!(
"workflow `{}` can nest into itself (a sub-workflow cycle) — runs are \
bounded by PICLOUD_WORKFLOW_MAX_DEPTH, not infinite, but this is likely a bug",
w.name
));
}
}
out
}
/// Per-kind structural validation for one desired trigger — kept at parity /// Per-kind structural validation for one desired trigger — kept at parity
/// with the interactive trigger API so `apply` can't write a trigger the /// with the interactive trigger API so `apply` can't write a trigger the
/// dashboard would have rejected. Pure (no live state), so it's unit-tested /// dashboard would have rejected. Pure (no live state), so it's unit-tested
@@ -6695,4 +6754,47 @@ mod tests {
])); ]));
assert!(validate_workflow_definition("w", &bad_tmpl).is_err()); assert!(validate_workflow_definition("w", &bad_tmpl).is_err());
} }
fn bundle_workflow(name: &str, steps: serde_json::Value) -> BundleWorkflow {
BundleWorkflow {
name: name.into(),
definition: wf_def(steps),
enabled: true,
}
}
#[test]
fn workflow_nesting_warning_flags_self_and_mutual_cycles() {
// Direct self-nest.
let mut b = empty_bundle();
b.workflows = vec![bundle_workflow(
"loop",
serde_json::json!([{ "name": "s", "workflow": "loop" }]),
)];
let w = workflow_nesting_warnings(&b);
assert_eq!(w.len(), 1);
assert!(w[0].contains("loop") && w[0].contains("cycle"));
// Mutual cycle w1 -> w2 -> w1.
let mut b = empty_bundle();
b.workflows = vec![
bundle_workflow("w1", serde_json::json!([{ "name": "s", "workflow": "w2" }])),
bundle_workflow("w2", serde_json::json!([{ "name": "s", "workflow": "w1" }])),
];
assert_eq!(workflow_nesting_warnings(&b).len(), 2);
// Acyclic nesting (parent -> child, child is a leaf) → no warning.
let mut b = empty_bundle();
b.workflows = vec![
bundle_workflow(
"parent",
serde_json::json!([{ "name": "s", "workflow": "child" }]),
),
bundle_workflow(
"child",
serde_json::json!([{ "name": "s", "function": "f" }]),
),
];
assert!(workflow_nesting_warnings(&b).is_empty());
}
} }

View File

@@ -27,9 +27,13 @@
//! A second, slower cadence reclaims steps leased by a crashed worker //! A second, slower cadence reclaims steps leased by a crashed worker
//! ([`workflow_repo::reclaim_stale_steps`]) — the durability safety net. //! ([`workflow_repo::reclaim_stale_steps`]) — the durability safety net.
//! //!
//! M2 executes **function** steps only; `workflow`-kind (nested sub-workflow) //! Each claimed step is *prepared* ([`Prep`]) — the run context is built, its
//! steps land in M4, and `when` conditions + input templating in M3. The seams //! `when` condition evaluated (false → `skipped`, M3), and its `input` template
//! are present (the step target enum, `run_input`, `workflow_depth`). //! resolved (M3) — then dispatched by target: a **function** runs through the
//! executor; a **`workflow`** starts a nested child run and parks the parent
//! step on it ([`workflow_repo::start_child_and_park`], M4), resumed by
//! [`workflow_repo::resume_finished_children`] once the child terminates. Nested
//! runs are bounded by [`WorkflowOrchestratorConfig::max_depth`].
//! //!
//! [`cron_scheduler`]: crate::cron_scheduler //! [`cron_scheduler`]: crate::cron_scheduler
@@ -42,19 +46,41 @@ use tokio::sync::OwnedSemaphorePermit;
use picloud_executor_core::{build_execution_log, ExecRequest, InvocationType}; use picloud_executor_core::{build_execution_log, ExecRequest, InvocationType};
use picloud_orchestrator_core::{ExecutionGate, ExecutorClient, ScriptIdentity}; use picloud_orchestrator_core::{ExecutionGate, ExecutorClient, ScriptIdentity};
use picloud_shared::workflow::{StepTarget, WorkflowBackoff, WorkflowStepDef}; use picloud_shared::workflow::{StepTarget, WorkflowBackoff, WorkflowDefinition, WorkflowStepDef};
use picloud_shared::{ExecutionId, ExecutionLogSink, ExecutionSource, RequestId}; use picloud_shared::{
ExecutionId, ExecutionLogSink, ExecutionSource, RequestId, Script, WorkflowId,
};
use crate::dispatcher::compute_backoff; use crate::dispatcher::compute_backoff;
use crate::repo::ScriptRepository; use crate::repo::ScriptRepository;
use crate::trigger_config::BackoffShape; use crate::trigger_config::BackoffShape;
use crate::workflow_expr; use crate::workflow_expr;
use crate::workflow_repo::{ use crate::workflow_repo::{
claim_ready_step, complete_step_and_advance, load_run_step_outputs, reclaim_stale_steps, claim_ready_step, complete_step_and_advance, get_workflow_by_name, load_run_step_outputs,
ClaimedStep, StepOutcome, reclaim_stale_steps, resume_finished_children, start_child_and_park, ClaimedStep, StepOutcome,
}; };
use crate::workflow_template::{self, RunContext}; use crate::workflow_template::{self, RunContext};
/// The prepared action for a claimed step, after context build + `when` + input
/// resolution + target lookup. The orchestrator dispatches on this.
enum Prep {
/// `when` evaluated false — skip the step (never executed).
Skip,
/// A resolution error, already turned into a `Failed` outcome.
Fail(StepOutcome),
/// A function step: run this script with this resolved body.
Function {
script: Box<Script>,
body: serde_json::Value,
},
/// A nested sub-workflow step: start this child run with this input.
StartChild {
child_id: WorkflowId,
child_def: WorkflowDefinition,
input: serde_json::Value,
},
}
/// Max steps claimed (and thus gate permits held) per orchestrator tick. Keeps /// Max steps claimed (and thus gate permits held) per orchestrator tick. Keeps
/// the workflow worker from monopolizing the gate it shares with HTTP + the /// the workflow worker from monopolizing the gate it shares with HTTP + the
/// dispatcher; the tick cadence + reclaim pick up the rest. /// dispatcher; the tick cadence + reclaim pick up the rest.
@@ -182,6 +208,15 @@ impl WorkflowOrchestrator {
pub async fn tick_once(&self) { pub async fn tick_once(&self) {
use futures::stream::{self, StreamExt}; use futures::stream::{self, StreamExt};
// M4: resolve any parent steps whose child sub-workflow has finished —
// first, so a just-completed child promotes the parent's dependents to
// `ready` in time for this tick's claim scan below.
match resume_finished_children(&self.pool).await {
Ok(0) => {}
Ok(n) => tracing::debug!(resolved = n, "workflow parent steps resumed"),
Err(e) => tracing::warn!(?e, "workflow child-resume errored"),
}
let mut batch: Vec<(ClaimedStep, OwnedSemaphorePermit)> = Vec::new(); let mut batch: Vec<(ClaimedStep, OwnedSemaphorePermit)> = Vec::new();
for _ in 0..STEP_CLAIM_BATCH { for _ in 0..STEP_CLAIM_BATCH {
// Acquire the gate slot BEFORE claiming so a claimed step always // Acquire the gate slot BEFORE claiming so a claimed step always
@@ -213,12 +248,50 @@ impl WorkflowOrchestrator {
.await; .await;
} }
/// Execute one claimed step, then write its outcome + advance the run. /// Process one claimed step: prepare it (context → `when` → input →
/// target), then dispatch — run a function, start a sub-workflow, skip, or
/// fail.
async fn run_step(&self, claimed: ClaimedStep, permit: OwnedSemaphorePermit) { async fn run_step(&self, claimed: ClaimedStep, permit: OwnedSemaphorePermit) {
let outcome = self.execute_step(&claimed).await; match self.prepare(&claimed).await {
// Release the gate slot before the (DB-only, fast) advance write. Prep::Skip => {
drop(permit); drop(permit);
if let Err(e) = complete_step_and_advance(&self.pool, &claimed, outcome).await { self.complete(&claimed, StepOutcome::Skipped).await;
}
Prep::Fail(outcome) => {
drop(permit);
self.complete(&claimed, outcome).await;
}
Prep::Function { script, body } => {
let outcome = self.execute_function(&claimed, &script, body).await;
// Release the gate slot before the (DB-only, fast) advance.
drop(permit);
self.complete(&claimed, outcome).await;
}
Prep::StartChild {
child_id,
child_def,
input,
} => {
// A nested start runs no script — release the gate slot; the
// child's own steps compete for it when they run.
drop(permit);
if let Err(e) =
start_child_and_park(&self.pool, &claimed, child_id, &child_def, input).await
{
tracing::error!(
?e,
run_id = %claimed.run_id,
step = %claimed.step_name,
"workflow nested-start write failed"
);
}
}
}
}
/// Write a step outcome + advance the run, logging any DB error.
async fn complete(&self, claimed: &ClaimedStep, outcome: StepOutcome) {
if let Err(e) = complete_step_and_advance(&self.pool, claimed, outcome).await {
tracing::error!( tracing::error!(
?e, ?e,
run_id = %claimed.run_id, run_id = %claimed.run_id,
@@ -228,17 +301,17 @@ impl WorkflowOrchestrator {
} }
} }
/// Resolve + run a step's function, returning the outcome. Failures are /// Prepare a claimed step: build the run context, evaluate `when`, resolve
/// converted to [`StepOutcome::Failed`] carrying the retry delay derived /// the input template, and resolve the target. Pure of side effects beyond
/// from the step's own retry policy (the repo decides retry-vs-terminal). /// the reads it needs; the caller dispatches on the returned [`Prep`].
#[allow(clippy::too_many_lines)] #[allow(clippy::too_many_lines)]
async fn execute_step(&self, claimed: &ClaimedStep) -> StepOutcome { async fn prepare(&self, claimed: &ClaimedStep) -> Prep {
let Some(step) = claimed.step_def() else { let Some(step) = claimed.step_def() else {
return self.fail( return Prep::Fail(self.fail(
None, None,
claimed, claimed,
"step not found in workflow definition".into(), "step not found in workflow definition".into(),
); ));
}; };
// M3: build the run context (run input + prior succeeded outputs) once; // M3: build the run context (run input + prior succeeded outputs) once;
@@ -246,7 +319,11 @@ impl WorkflowOrchestrator {
let outputs = match load_run_step_outputs(&self.pool, claimed.run_id).await { let outputs = match load_run_step_outputs(&self.pool, claimed.run_id).await {
Ok(o) => o, Ok(o) => o,
Err(e) => { Err(e) => {
return self.fail(Some(step), claimed, format!("loading step context: {e}")); return Prep::Fail(self.fail(
Some(step),
claimed,
format!("loading step context: {e}"),
));
} }
}; };
let ctx = RunContext { let ctx = RunContext {
@@ -254,84 +331,125 @@ impl WorkflowOrchestrator {
steps: outputs, steps: outputs,
}; };
// M3: a false `when` condition skips the step — never executed, and // M3: a false `when` skips the step — never executed, satisfied-but-empty.
// satisfied-but-empty for its dependents (`compute_advance`).
if let Some(cond) = &step.when { if let Some(cond) = &step.when {
match workflow_expr::eval(cond, &ctx) { match workflow_expr::eval(cond, &ctx) {
Ok(true) => {} Ok(true) => {}
Ok(false) => return StepOutcome::Skipped, Ok(false) => return Prep::Skip,
Err(e) => { Err(e) => {
return self.fail(Some(step), claimed, format!("evaluating `when`: {e}")); return Prep::Fail(self.fail(
Some(step),
claimed,
format!("evaluating `when`: {e}"),
));
} }
} }
} }
let fn_name = match step.target() { // M3: resolve `{{ … }}` references in the step input. Null passes the
Some(StepTarget::Function(f)) => f.to_string(), // run input through; a missing reference is a hard failure.
Some(StepTarget::Workflow(_)) => {
// Nested sub-workflows land in M4.
return self.fail(
Some(step),
claimed,
"sub-workflow steps not yet supported".into(),
);
}
None => {
return self.fail(
Some(step),
claimed,
"step declares neither `function` nor `workflow`".into(),
);
}
};
// Resolve the function by name in the RUN's app scope (an inherited
// group script is reachable, nearest-owner-wins) — never off a
// script-passed arg. cx isolation boundary preserved.
let script = match self
.scripts
.get_by_name_inherited(claimed.app_id, &fn_name)
.await
{
Ok(Some(s)) if s.enabled => s,
Ok(Some(_)) => {
return self.fail(
Some(step),
claimed,
format!("function {fn_name:?} is disabled"),
);
}
Ok(None) => {
return self.fail(
Some(step),
claimed,
format!("function {fn_name:?} not found"),
);
}
Err(e) => {
return self.fail(Some(step), claimed, format!("resolving {fn_name:?}: {e}"));
}
};
// M3: resolve `{{ … }}` references in the step input against the
// context. A null input passes the run input through unchanged; a
// missing reference is a hard step failure (a definition bug we
// surface, not hide).
let body = if step.input.is_null() { let body = if step.input.is_null() {
claimed.run_input.clone() claimed.run_input.clone()
} else { } else {
match workflow_template::resolve(&step.input, &ctx) { match workflow_template::resolve(&step.input, &ctx) {
Ok(v) => v, Ok(v) => v,
Err(e) => { Err(e) => {
return self.fail(Some(step), claimed, format!("resolving input: {e}")); return Prep::Fail(self.fail(
Some(step),
claimed,
format!("resolving input: {e}"),
));
} }
} }
}; };
let execution_id = ExecutionId::new(); match step.target() {
Some(StepTarget::Function(fn_name)) => {
// Resolve the function by name in the RUN's app scope (an
// inherited group script is reachable, nearest-owner-wins) —
// never off a script-passed arg. cx isolation boundary held.
match self
.scripts
.get_by_name_inherited(claimed.app_id, fn_name)
.await
{
Ok(Some(s)) if s.enabled => Prep::Function {
script: Box::new(s),
body,
},
Ok(Some(_)) => Prep::Fail(self.fail(
Some(step),
claimed,
format!("function {fn_name:?} is disabled"),
)),
Ok(None) => Prep::Fail(self.fail(
Some(step),
claimed,
format!("function {fn_name:?} not found"),
)),
Err(e) => Prep::Fail(self.fail(
Some(step),
claimed,
format!("resolving {fn_name:?}: {e}"),
)),
}
}
Some(StepTarget::Workflow(wf_name)) => {
// M4: nesting depth ceiling — the child would run at depth + 1.
let ceiling = i32::try_from(self.config.max_depth).unwrap_or(i32::MAX);
if claimed.workflow_depth + 1 > ceiling {
return Prep::Fail(self.fail(
Some(step),
claimed,
format!(
"max workflow nesting depth ({}) exceeded",
self.config.max_depth
),
));
}
match get_workflow_by_name(&self.pool, claimed.app_id, wf_name).await {
Ok(Some(w)) if w.enabled => Prep::StartChild {
child_id: w.id,
child_def: w.definition,
input: body,
},
Ok(Some(_)) => Prep::Fail(self.fail(
Some(step),
claimed,
format!("workflow {wf_name:?} is disabled"),
)),
Ok(None) => Prep::Fail(self.fail(
Some(step),
claimed,
format!("workflow {wf_name:?} not found"),
)),
Err(e) => Prep::Fail(self.fail(
Some(step),
claimed,
format!("resolving workflow {wf_name:?}: {e}"),
)),
}
}
None => Prep::Fail(self.fail(
Some(step),
claimed,
"step declares neither `function` nor `workflow`".into(),
)),
}
}
/// Run a resolved function step through the executor, returning its outcome
/// (a failure carries the step's retry-policy backoff). Logs the run under
/// `ExecutionSource::Workflow` so `pic logs` surfaces it.
async fn execute_function(
&self,
claimed: &ClaimedStep,
script: &Script,
body: serde_json::Value,
) -> StepOutcome {
let request_id = RequestId::new(); let request_id = RequestId::new();
let req = ExecRequest { let req = ExecRequest {
execution_id, execution_id: ExecutionId::new(),
request_id, request_id,
script_id: script.id, script_id: script.id,
script_name: script.name.clone(), script_name: script.name.clone(),
@@ -346,8 +464,8 @@ impl WorkflowOrchestrator {
sandbox_overrides: script.sandbox, sandbox_overrides: script.sandbox,
app_id: claimed.app_id, app_id: claimed.app_id,
script_owner: script.owner(), script_owner: script.owner(),
// Like invoke_async / HTTP outbox rows: steps run with no // Like invoke_async / HTTP outbox rows: steps run with no principal
// principal (the run's origin is forensic, not re-authenticated). // (the run's origin is forensic, not re-authenticated).
principal: None, principal: None,
trigger_depth: 0, trigger_depth: 0,
root_execution_id: ExecutionId::from(claimed.root_execution_id), root_execution_id: ExecutionId::from(claimed.root_execution_id),
@@ -366,7 +484,6 @@ impl WorkflowOrchestrator {
.await; .await;
let finished = Utc::now(); let finished = Utc::now();
// Log the run so `pic logs` surfaces workflow steps (source=workflow).
let log = build_execution_log( let log = build_execution_log(
claimed.app_id, claimed.app_id,
script.id, script.id,
@@ -385,7 +502,7 @@ impl WorkflowOrchestrator {
match outcome { match outcome {
Ok(resp) => StepOutcome::Succeeded(resp.body), Ok(resp) => StepOutcome::Succeeded(resp.body),
Err(err) => self.fail(Some(step), claimed, err.to_string()), Err(err) => self.fail(claimed.step_def(), claimed, err.to_string()),
} }
} }

View File

@@ -461,6 +461,19 @@ pub async fn start_run(
definition: &WorkflowDefinition, definition: &WorkflowDefinition,
) -> Result<WorkflowRunId, WorkflowRepoError> { ) -> Result<WorkflowRunId, WorkflowRepoError> {
let mut tx = pool.begin().await?; let mut tx = pool.begin().await?;
let run_id = seed_run_tx(&mut tx, &new, definition).await?;
tx.commit().await?;
Ok(run_id)
}
/// Insert the `workflow_runs` row + its `workflow_run_steps` (roots `ready`,
/// the rest `pending`) inside an open transaction. Shared by `start_run` and
/// the nested-child start (`start_child_and_park`).
pub async fn seed_run_tx(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
new: &NewWorkflowRun,
definition: &WorkflowDefinition,
) -> Result<WorkflowRunId, WorkflowRepoError> {
let (run_id,): (Uuid,) = sqlx::query_as( let (run_id,): (Uuid,) = sqlx::query_as(
"INSERT INTO workflow_runs \ "INSERT INTO workflow_runs \
(workflow_id, app_id, status, input, root_execution_id, \ (workflow_id, app_id, status, input, root_execution_id, \
@@ -474,7 +487,7 @@ pub async fn start_run(
.bind(new.workflow_depth) .bind(new.workflow_depth)
.bind(new.parent_run_id.map(WorkflowRunId::into_inner)) .bind(new.parent_run_id.map(WorkflowRunId::into_inner))
.bind(new.parent_step_id.map(WorkflowRunStepId::into_inner)) .bind(new.parent_step_id.map(WorkflowRunStepId::into_inner))
.fetch_one(&mut *tx) .fetch_one(&mut **tx)
.await?; .await?;
for step in &definition.steps { for step in &definition.steps {
@@ -493,11 +506,10 @@ pub async fn start_run(
.bind(&step.name) .bind(&step.name)
.bind(status) .bind(status)
.bind(max_attempts) .bind(max_attempts)
.execute(&mut *tx) .execute(&mut **tx)
.await?; .await?;
} }
tx.commit().await?;
Ok(run_id.into()) Ok(run_id.into())
} }
@@ -683,28 +695,43 @@ pub async fn complete_step_and_advance(
} }
} }
// 2. Advance — lock the run row so concurrent advances of the same run // 2. Advance the run graph in the same tx.
// serialize (each does a full recompute, so this is idempotent). advance_run_tx(&mut tx, claimed.run_id, &claimed.definition).await?;
tx.commit().await?;
Ok(AdvanceResult::Advanced)
}
/// Advance one run's graph inside an open transaction: lock the run row (so
/// concurrent advances serialize — each does a full recompute, so this is
/// idempotent), promote pending steps whose deps are now satisfied, and write
/// the run's terminal status. A no-op if the run is already terminal. Does NOT
/// commit — the caller owns the transaction.
///
/// Shared by `complete_step_and_advance` (a function/skip step finished) and
/// `resume_finished_children` (a parked sub-workflow step finished).
pub async fn advance_run_tx(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
run_id: WorkflowRunId,
definition: &WorkflowDefinition,
) -> Result<(), WorkflowRepoError> {
let run: Option<(String,)> = let run: Option<(String,)> =
sqlx::query_as("SELECT status FROM workflow_runs WHERE id = $1 FOR UPDATE") sqlx::query_as("SELECT status FROM workflow_runs WHERE id = $1 FOR UPDATE")
.bind(claimed.run_id.into_inner()) .bind(run_id.into_inner())
.fetch_optional(&mut *tx) .fetch_optional(&mut **tx)
.await?; .await?;
let Some((run_status,)) = run else { let Some((run_status,)) = run else {
tx.commit().await?; return Ok(());
return Ok(AdvanceResult::Advanced);
}; };
if RunStatus::from_str(&run_status).is_some_and(RunStatus::is_terminal) { if RunStatus::from_str(&run_status).is_some_and(RunStatus::is_terminal) {
// Another worker already finished the run; nothing to do. // Another worker already finished the run; nothing to do.
tx.commit().await?; return Ok(());
return Ok(AdvanceResult::Advanced);
} }
let step_rows: Vec<StepStateRow> = sqlx::query_as( let step_rows: Vec<StepStateRow> = sqlx::query_as(
"SELECT step_name, status, output FROM workflow_run_steps WHERE run_id = $1", "SELECT step_name, status, output FROM workflow_run_steps WHERE run_id = $1",
) )
.bind(claimed.run_id.into_inner()) .bind(run_id.into_inner())
.fetch_all(&mut *tx) .fetch_all(&mut **tx)
.await?; .await?;
let mut statuses = BTreeMap::new(); let mut statuses = BTreeMap::new();
let mut outputs = BTreeMap::new(); let mut outputs = BTreeMap::new();
@@ -717,7 +744,7 @@ pub async fn complete_step_and_advance(
} }
} }
let plan = compute_advance(&claimed.definition, &statuses, &outputs); let plan = compute_advance(definition, &statuses, &outputs);
for name in &plan.promote { for name in &plan.promote {
sqlx::query( sqlx::query(
@@ -725,9 +752,9 @@ pub async fn complete_step_and_advance(
SET status = 'ready', next_attempt_at = NULL, updated_at = NOW() \ SET status = 'ready', next_attempt_at = NULL, updated_at = NOW() \
WHERE run_id = $1 AND step_name = $2 AND status = 'pending'", WHERE run_id = $1 AND step_name = $2 AND status = 'pending'",
) )
.bind(claimed.run_id.into_inner()) .bind(run_id.into_inner())
.bind(name) .bind(name)
.execute(&mut *tx) .execute(&mut **tx)
.await?; .await?;
} }
@@ -737,9 +764,9 @@ pub async fn complete_step_and_advance(
"UPDATE workflow_runs \ "UPDATE workflow_runs \
SET status = 'succeeded', output = $2, finished_at = NOW() WHERE id = $1", SET status = 'succeeded', output = $2, finished_at = NOW() WHERE id = $1",
) )
.bind(claimed.run_id.into_inner()) .bind(run_id.into_inner())
.bind(plan.run_output) .bind(plan.run_output)
.execute(&mut *tx) .execute(&mut **tx)
.await?; .await?;
} }
RunStatus::Failed => { RunStatus::Failed => {
@@ -747,9 +774,9 @@ pub async fn complete_step_and_advance(
"UPDATE workflow_runs \ "UPDATE workflow_runs \
SET status = 'failed', error = $2, finished_at = NOW() WHERE id = $1", SET status = 'failed', error = $2, finished_at = NOW() WHERE id = $1",
) )
.bind(claimed.run_id.into_inner()) .bind(run_id.into_inner())
.bind(plan.run_error) .bind(plan.run_error)
.execute(&mut *tx) .execute(&mut **tx)
.await?; .await?;
} }
_ => { _ => {
@@ -757,14 +784,13 @@ pub async fn complete_step_and_advance(
"UPDATE workflow_runs \ "UPDATE workflow_runs \
SET status = 'running', started_at = COALESCE(started_at, NOW()) WHERE id = $1", SET status = 'running', started_at = COALESCE(started_at, NOW()) WHERE id = $1",
) )
.bind(claimed.run_id.into_inner()) .bind(run_id.into_inner())
.execute(&mut *tx) .execute(&mut **tx)
.await?; .await?;
} }
} }
tx.commit().await?; Ok(())
Ok(AdvanceResult::Advanced)
} }
/// Periodic safety net: a step leased by a crashed worker (still `running` /// Periodic safety net: a step leased by a crashed worker (still `running`
@@ -924,6 +950,161 @@ pub async fn load_run_step_outputs(
.collect()) .collect())
} }
// ---- nested sub-workflows (M4) --------------------------------------------
/// Resolve an app-owned workflow by case-insensitive name (a `workflow`-kind
/// step's target). Pool free-fn mirror of `WorkflowRepo::get_by_name`.
pub async fn get_workflow_by_name(
pool: &PgPool,
app_id: AppId,
name: &str,
) -> Result<Option<Workflow>, WorkflowRepoError> {
let row: Option<WorkflowRow> = sqlx::query_as(&format!(
"SELECT {SELECT_COLS} FROM workflows WHERE app_id = $1 AND LOWER(name) = LOWER($2)"
))
.bind(app_id.into_inner())
.bind(name)
.fetch_optional(pool)
.await?;
row.map(TryInto::try_into).transpose()
}
/// Start a nested sub-workflow run and **park** the parent step on it. In one
/// token-gated transaction: the parent step (still claimed by the caller) is
/// set `running` with `claim_token` cleared and `child_run_id` pointing at a
/// freshly-seeded child run (depth + 1, parent linkage). The parked step is
/// never re-claimed (only `ready` steps are) nor reclaimed (only leased
/// `running` steps — `claim_token` is NULL here); `resume_finished_children`
/// resolves it once the child terminates. A stale worker matches zero rows and
/// nothing (including the child) is written.
pub async fn start_child_and_park(
pool: &PgPool,
claimed: &ClaimedStep,
child_workflow_id: WorkflowId,
child_definition: &WorkflowDefinition,
child_input: serde_json::Value,
) -> Result<AdvanceResult, WorkflowRepoError> {
let mut tx = pool.begin().await?;
// Token-gated park FIRST — so a stale claim short-circuits before we ever
// insert an orphan child run (child_run_id is linked below).
let res = sqlx::query(
"UPDATE workflow_run_steps \
SET status = 'running', claim_token = NULL, claimed_at = NULL, updated_at = NOW() \
WHERE id = $1 AND claim_token = $2",
)
.bind(claimed.step_id.into_inner())
.bind(claimed.claim_token)
.execute(&mut *tx)
.await?;
if res.rows_affected() == 0 {
tx.rollback().await?;
return Ok(AdvanceResult::Stale);
}
let new = NewWorkflowRun {
workflow_id: child_workflow_id,
app_id: claimed.app_id,
input: child_input,
// Correlate the whole nested tree under the top-level run's root id.
root_execution_id: claimed.root_execution_id,
workflow_depth: claimed.workflow_depth + 1,
parent_run_id: Some(claimed.run_id),
parent_step_id: Some(claimed.step_id),
};
let child_run = seed_run_tx(&mut tx, &new, child_definition).await?;
sqlx::query(
"UPDATE workflow_run_steps SET child_run_id = $2, updated_at = NOW() WHERE id = $1",
)
.bind(claimed.step_id.into_inner())
.bind(child_run.into_inner())
.execute(&mut *tx)
.await?;
sqlx::query(
"UPDATE workflow_runs SET status = 'running', started_at = COALESCE(started_at, NOW()) \
WHERE id = $1 AND status = 'pending'",
)
.bind(claimed.run_id.into_inner())
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(AdvanceResult::Advanced)
}
#[derive(sqlx::FromRow)]
struct ParkedRow {
parent_step_id: Uuid,
parent_run_id: Uuid,
child_status: String,
child_output: Option<serde_json::Value>,
child_error: Option<String>,
parent_definition: serde_json::Value,
}
/// Resolve parent steps parked on a now-terminal child sub-workflow: write the
/// child's output (or error) onto the parent step and advance the parent run.
/// Idempotent — the conditional `status = 'running'` gate means a second pass
/// (or a second worker) matches zero rows. Returns the number resolved.
pub async fn resume_finished_children(pool: &PgPool) -> Result<u64, WorkflowRepoError> {
let rows: Vec<ParkedRow> = sqlx::query_as(
"SELECT ps.id AS parent_step_id, ps.run_id AS parent_run_id, \
child.status AS child_status, child.output AS child_output, \
child.error AS child_error, w.definition AS parent_definition \
FROM workflow_run_steps ps \
JOIN workflow_runs child ON child.id = ps.child_run_id \
JOIN workflow_runs parent ON parent.id = ps.run_id \
JOIN workflows w ON w.id = parent.workflow_id \
WHERE ps.status = 'running' AND ps.child_run_id IS NOT NULL \
AND ps.claim_token IS NULL \
AND child.status IN ('succeeded', 'failed', 'canceled') \
LIMIT 32",
)
.fetch_all(pool)
.await?;
let mut resolved = 0u64;
for r in rows {
let mut tx = pool.begin().await?;
let (new_status, out, err) = if r.child_status == "succeeded" {
("succeeded", r.child_output.clone(), None)
} else {
(
"failed",
None,
Some(
r.child_error
.clone()
.unwrap_or_else(|| "sub-workflow failed".to_string()),
),
)
};
let res = sqlx::query(
"UPDATE workflow_run_steps \
SET status = $2, output = $3, error = $4, updated_at = NOW() \
WHERE id = $1 AND status = 'running' AND child_run_id IS NOT NULL",
)
.bind(r.parent_step_id)
.bind(new_status)
.bind(&out)
.bind(&err)
.execute(&mut *tx)
.await?;
if res.rows_affected() == 0 {
tx.rollback().await?;
continue; // already resolved by another pass
}
let parent_def: WorkflowDefinition = serde_json::from_value(r.parent_definition)
.map_err(|e| WorkflowRepoError::Serde(e.to_string()))?;
advance_run_tx(&mut tx, r.parent_run_id.into(), &parent_def).await?;
tx.commit().await?;
resolved += 1;
}
Ok(resolved)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

View File

@@ -616,12 +616,13 @@ async fn seed_scripts(pool: &PgPool, app: AppId, names: &[&str]) {
} }
} }
/// Drive a real orchestrator to terminal using `executor`. /// Drive a real orchestrator to terminal using `executor` + `config`.
async fn drive_to_terminal( async fn drive_with(
pool: &PgPool, pool: &PgPool,
executor: Arc<dyn ExecutorClient>, executor: Arc<dyn ExecutorClient>,
app: AppId, app: AppId,
run: picloud_shared::WorkflowRunId, run: picloud_shared::WorkflowRunId,
config: WorkflowOrchestratorConfig,
) { ) {
let orch = WorkflowOrchestrator { let orch = WorkflowOrchestrator {
pool: pool.clone(), pool: pool.clone(),
@@ -629,9 +630,9 @@ async fn drive_to_terminal(
executor, executor,
gate: Arc::new(ExecutionGate::new(8)), gate: Arc::new(ExecutionGate::new(8)),
log_sink: Arc::new(NoopLogSink), log_sink: Arc::new(NoopLogSink),
config: WorkflowOrchestratorConfig::default(), config,
}; };
for _ in 0..50 { for _ in 0..80 {
orch.tick_once().await; orch.tick_once().await;
if get_run(pool, app, run) if get_run(pool, app, run)
.await .await
@@ -645,6 +646,76 @@ async fn drive_to_terminal(
} }
} }
/// Drive to terminal with the default config.
async fn drive_to_terminal(
pool: &PgPool,
executor: Arc<dyn ExecutorClient>,
app: AppId,
run: picloud_shared::WorkflowRunId,
) {
drive_with(
pool,
executor,
app,
run,
WorkflowOrchestratorConfig::default(),
)
.await;
}
/// A `workflow`-kind (nested sub-workflow) step.
fn wf_step(name: &str, workflow: &str, deps: &[&str]) -> WorkflowStepDef {
WorkflowStepDef {
name: name.into(),
function: None,
workflow: Some(workflow.into()),
input: Value::Null,
depends_on: deps.iter().map(|s| (*s).to_string()).collect(),
when: None,
retry: None,
on_error: OnError::Fail,
}
}
/// Create just a group + app; return the app id.
async fn seed_app(pool: &PgPool) -> AppId {
let sfx = Uuid::new_v4().simple().to_string();
let grp: (Uuid,) =
sqlx::query_as("INSERT INTO groups (slug, name) VALUES ($1, $1) RETURNING id")
.bind(format!("wf-grp-{sfx}"))
.fetch_one(pool)
.await
.expect("insert group");
let app: (Uuid,) =
sqlx::query_as("INSERT INTO apps (slug, name, group_id) VALUES ($1, $1, $2) RETURNING id")
.bind(format!("wf-app-{sfx}"))
.bind(grp.0)
.fetch_one(pool)
.await
.expect("insert app");
app.0.into()
}
/// Insert a workflow with a specific name into an existing app.
async fn seed_named_workflow(
pool: &PgPool,
app: AppId,
name: &str,
def: &WorkflowDefinition,
) -> WorkflowId {
let def_json = serde_json::to_value(def).unwrap();
let wf: (Uuid,) = sqlx::query_as(
"INSERT INTO workflows (app_id, name, definition) VALUES ($1, $2, $3) RETURNING id",
)
.bind(app.into_inner())
.bind(name)
.bind(def_json)
.fetch_one(pool)
.await
.expect("insert named workflow");
wf.0.into()
}
fn run_with_input(app: AppId, wf: WorkflowId, input: Value) -> NewWorkflowRun { fn run_with_input(app: AppId, wf: WorkflowId, input: Value) -> NewWorkflowRun {
NewWorkflowRun { NewWorkflowRun {
workflow_id: wf, workflow_id: wf,
@@ -762,3 +833,97 @@ async fn missing_input_ref_fails_the_step() {
assert_eq!(bstep.status, StepStatus::Failed); assert_eq!(bstep.status, StepStatus::Failed);
assert!(bstep.error.as_deref().unwrap_or("").contains("input")); assert!(bstep.error.as_deref().unwrap_or("").contains("input"));
} }
// ===========================================================================
// M4 — nested sub-workflows.
// ===========================================================================
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn nested_sub_workflow_runs_and_output_flows_to_parent() {
let Some(pool) = pool_or_skip().await else {
return;
};
let _claim_guard = lock_and_reset(&pool).await;
let app = seed_app(&pool).await;
// Child workflow: one function step producing { answer: 42 }.
let child_def = WorkflowDefinition {
steps: vec![step("leaf", "fn_leaf", &[])],
};
seed_named_workflow(&pool, app, "child", &child_def).await;
// Parent: a `workflow`-kind step "call" → child, then "after" consuming its
// output. The child run's output is { leaf: <leaf output> }, so
// steps.call.output.leaf.answer reaches through the nesting.
let mut after = step("after", "fn_after", &["call"]);
after.input = json!({ "got": "{{ steps.call.output.leaf.answer }}" });
let parent_def = WorkflowDefinition {
steps: vec![wf_step("call", "child", &[]), after],
};
let parent_wf = seed_named_workflow(&pool, app, "parent", &parent_def).await;
seed_scripts(&pool, app, &["fn_leaf", "fn_after"]).await;
let run = start_run(&pool, new_run(app, parent_wf), &parent_def)
.await
.expect("start");
let executor = Arc::new(ScriptedExecutor::new(&[(
"fn_leaf",
json!({ "answer": 42 }),
)]));
drive_with(
&pool,
executor.clone(),
app,
run,
WorkflowOrchestratorConfig::default(),
)
.await;
let r = get_run(&pool, app, run).await.unwrap().unwrap();
assert_eq!(r.status, RunStatus::Succeeded);
// The child's output flowed into the parent step, then into `after`'s input.
assert_eq!(executor.body_for("fn_after"), Some(json!({ "got": 42 })));
// The parent step "call" carries the child run's output.
let steps = list_run_steps(&pool, run).await.unwrap();
let call = steps.iter().find(|s| s.step_name == "call").unwrap();
assert_eq!(call.status, StepStatus::Succeeded);
assert_eq!(call.output, Some(json!({ "leaf": { "answer": 42 } })));
assert!(
call.child_run_id.is_some(),
"parent step links its child run"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn nesting_depth_ceiling_fails_the_run() {
let Some(pool) = pool_or_skip().await else {
return;
};
let _claim_guard = lock_and_reset(&pool).await;
let app = seed_app(&pool).await;
// A workflow that nests into ITSELF — bounded only by the depth ceiling.
let def = WorkflowDefinition {
steps: vec![wf_step("recurse", "loopwf", &[])],
};
let wf = seed_named_workflow(&pool, app, "loopwf", &def).await;
let run = start_run(&pool, new_run(app, wf), &def)
.await
.expect("start");
// max_depth = 2 → depth 0 and 1 spawn children; depth 2's spawn is refused.
let config = WorkflowOrchestratorConfig {
max_depth: 2,
..WorkflowOrchestratorConfig::default()
};
let executor = Arc::new(ScriptedExecutor::new(&[]));
drive_with(&pool, executor, app, run, config).await;
let r = get_run(&pool, app, run).await.unwrap().unwrap();
assert_eq!(
r.status,
RunStatus::Failed,
"self-nesting run fails at the depth ceiling, not infinitely"
);
}