feat: declarative project-tool foundation (pull/plan/apply/prune)
Add a server-side, atomic, declarative reconcile loop for a single app —
the foundation of the project-tool design. Developers describe an app's
scripts, routes, triggers, and secret-names in `picloud.toml`, then
`pic pull / plan / apply [--prune]` to converge live state to the manifest.
Server (manager-core):
- apply_service: a pure diff engine (compute_diff) shared by plan and
apply, plus an ApplyService that composes the existing per-repo writes
into ONE Postgres transaction. Identity keys mirror the DB UNIQUE
constraints (script=lower(name); route=(method,host_kind,host,
path_kind,path); trigger=per-kind semantic tuple; secret=name).
Apply takes a per-app advisory lock, recomputes the diff in-tx, applies
scripts -> routes -> triggers, prunes dependents-first, commits, then
refreshes the route table once post-commit.
- apply_api: POST /apps/{id}/plan (AppRead) and /apps/{id}/apply.
Apply requires the per-kind write caps the bundle exercises (all three
when --prune), plus AppSecretsRead when it binds an email trigger.
- tx-accepting repo siblings (insert/update/delete *_tx) so the existing
create/update/delete delegate to one SQL definition each.
- email triggers reference an inbound secret by NAME; the value is
resolved, decrypted (AAD-bound), and re-sealed server-side at apply —
it never travels in the manifest.
CLI (picloud-cli):
- manifest.rs (picloud.toml model), client plan/apply, and the pull/plan/
apply commands. pull rejects filesystem-unsafe script names up front.
Safety properties enforced and tested:
- idempotent: a freshly-pulled manifest re-applies as all-NoOp.
- atomic: a mid-bundle failure rolls back with nothing written.
- routes delete-before-insert so a freed binding is reusable in one apply.
- queue one-consumer invariant held inside the shared tx.
- email triggers are never pruned, and a script that still owns an
email/dead-letter trigger can't be pruned (the FK cascade would destroy
the sealed secret) — refused with a pointer to `pic triggers rm`.
- plan and apply agree on unset email-secret references.
No migration: the existing schema's UNIQUE constraints serve as identity
keys. Groups, env-scoping, and the `enabled` toggle are later milestones.
Tested: manager-core lib (360) + CLI bins (27) + 8 project-tool journeys
(pull/plan/apply/prune/email+queue), all green; clippy -D warnings clean.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
285
crates/picloud-cli/src/cmds/pull.rs
Normal file
285
crates/picloud-cli/src/cmds/pull.rs
Normal file
@@ -0,0 +1,285 @@
|
||||
//! `pic pull <app> [--dir .]` — export an app's current server state into
|
||||
//! a `picloud.toml` manifest plus `scripts/<name>.rhai` source files, for
|
||||
//! declarative management with `pic plan` / `pic apply`.
|
||||
//!
|
||||
//! Read-only: issues `GET`s only and writes local files. Every trigger
|
||||
//! kind is exported except `email` — the server stores the *sealed secret
|
||||
//! value*, not the secret name, so the manifest's `inbound_secret_ref`
|
||||
//! can't be reconstructed (email triggers must be set up by hand).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use picloud_shared::{DispatchMode, DocsEventOp, FilesEventOp, KvEventOp, ScriptId};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::client::Client;
|
||||
use crate::config;
|
||||
use crate::manifest::{
|
||||
CronTriggerSpec, DocsTriggerSpec, FilesTriggerSpec, KvTriggerSpec, Manifest, ManifestApp,
|
||||
ManifestRoute, ManifestScript, ManifestSecrets, ManifestTriggers, PubsubTriggerSpec,
|
||||
QueueTriggerSpec, MANIFEST_FILE,
|
||||
};
|
||||
use crate::output::{KvBlock, OutputMode};
|
||||
|
||||
pub async fn run(app_ident: &str, dir: &Path, mode: OutputMode) -> Result<()> {
|
||||
let creds = config::resolve()?;
|
||||
let client = Client::from_creds(&creds)?;
|
||||
|
||||
// One GET per resource kind (routes are per-script, below).
|
||||
let app = client.apps_get(app_ident).await?;
|
||||
let scripts = client.scripts_list_by_app(app_ident).await?;
|
||||
let triggers = client.triggers_list(app_ident).await?.triggers;
|
||||
let secrets = client.secrets_list(app_ident).await?.secrets;
|
||||
|
||||
let name_by_id: HashMap<ScriptId, String> =
|
||||
scripts.iter().map(|s| (s.id, s.name.clone())).collect();
|
||||
|
||||
// Routes: the admin surface lists them per script.
|
||||
let mut routes = Vec::new();
|
||||
for s in &scripts {
|
||||
for r in client.routes_list_for_script(&s.id.to_string()).await? {
|
||||
routes.push(ManifestRoute {
|
||||
script: s.name.clone(),
|
||||
method: r.method,
|
||||
host_kind: r.host_kind,
|
||||
host: r.host,
|
||||
host_param_name: r.host_param_name,
|
||||
path_kind: r.path_kind,
|
||||
path: r.path,
|
||||
dispatch_mode: r.dispatch_mode,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// The server does not constrain script names to a filesystem-safe
|
||||
// charset, so a name containing a path separator or `..` would let `pull`
|
||||
// write outside the project dir. Validate ALL names up front, before any
|
||||
// file is written, so a single bad name can't leave a half-written dir.
|
||||
// Reject rather than sanitize: a silent rename would desync the manifest
|
||||
// `name` from its `file`.
|
||||
for s in &scripts {
|
||||
if !is_safe_filename(&s.name) {
|
||||
anyhow::bail!(
|
||||
"script name {:?} is not filesystem-safe (contains a path \
|
||||
separator, `..`, or a leading dot); cannot pull",
|
||||
s.name
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Scripts: write each source next to the manifest and record a path ref.
|
||||
let scripts_dir = dir.join("scripts");
|
||||
std::fs::create_dir_all(&scripts_dir)
|
||||
.with_context(|| format!("creating {}", scripts_dir.display()))?;
|
||||
let mut manifest_scripts = Vec::with_capacity(scripts.len());
|
||||
for s in &scripts {
|
||||
let rel = format!("scripts/{}.rhai", s.name);
|
||||
std::fs::write(dir.join(&rel), &s.source).with_context(|| format!("writing {rel}"))?;
|
||||
manifest_scripts.push(ManifestScript {
|
||||
name: s.name.clone(),
|
||||
file: rel,
|
||||
kind: s.kind,
|
||||
description: s.description.clone(),
|
||||
timeout_seconds: i32::try_from(s.timeout_seconds).ok(),
|
||||
memory_limit_mb: i32::try_from(s.memory_limit_mb).ok(),
|
||||
sandbox: if s.sandbox.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s.sandbox)
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Triggers: map the five settled kinds; warn + skip the rest.
|
||||
let mut manifest_triggers = ManifestTriggers::default();
|
||||
let mut skipped: Vec<String> = Vec::new();
|
||||
for t in &triggers {
|
||||
let script = name_by_id
|
||||
.get(&t.script_id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| t.script_id.to_string());
|
||||
let dispatch_mode = DispatchMode::from_wire(&t.dispatch_mode);
|
||||
let retry_max_attempts = Some(t.retry_max_attempts);
|
||||
match t.kind.as_str() {
|
||||
"kv" => {
|
||||
let d: CollectionDetails<KvEventOp> = decode_details(&t.details, &t.kind)?;
|
||||
manifest_triggers.kv.push(KvTriggerSpec {
|
||||
script,
|
||||
collection_glob: d.collection_glob,
|
||||
ops: d.ops,
|
||||
dispatch_mode,
|
||||
retry_max_attempts,
|
||||
});
|
||||
}
|
||||
"docs" => {
|
||||
let d: CollectionDetails<DocsEventOp> = decode_details(&t.details, &t.kind)?;
|
||||
manifest_triggers.docs.push(DocsTriggerSpec {
|
||||
script,
|
||||
collection_glob: d.collection_glob,
|
||||
ops: d.ops,
|
||||
dispatch_mode,
|
||||
retry_max_attempts,
|
||||
});
|
||||
}
|
||||
"files" => {
|
||||
let d: CollectionDetails<FilesEventOp> = decode_details(&t.details, &t.kind)?;
|
||||
manifest_triggers.files.push(FilesTriggerSpec {
|
||||
script,
|
||||
collection_glob: d.collection_glob,
|
||||
ops: d.ops,
|
||||
dispatch_mode,
|
||||
retry_max_attempts,
|
||||
});
|
||||
}
|
||||
"cron" => {
|
||||
let d: CronDetails = decode_details(&t.details, &t.kind)?;
|
||||
manifest_triggers.cron.push(CronTriggerSpec {
|
||||
script,
|
||||
schedule: d.schedule,
|
||||
timezone: d.timezone,
|
||||
dispatch_mode,
|
||||
retry_max_attempts,
|
||||
});
|
||||
}
|
||||
"pubsub" => {
|
||||
let d: PubsubDetails = decode_details(&t.details, &t.kind)?;
|
||||
manifest_triggers.pubsub.push(PubsubTriggerSpec {
|
||||
script,
|
||||
topic_pattern: d.topic_pattern,
|
||||
dispatch_mode,
|
||||
retry_max_attempts,
|
||||
});
|
||||
}
|
||||
"queue" => {
|
||||
let d: QueueDetails = decode_details(&t.details, &t.kind)?;
|
||||
manifest_triggers.queue.push(QueueTriggerSpec {
|
||||
script,
|
||||
queue_name: d.queue_name,
|
||||
visibility_timeout_secs: Some(d.visibility_timeout_secs),
|
||||
dispatch_mode,
|
||||
retry_max_attempts,
|
||||
});
|
||||
}
|
||||
// `email` is skipped: the server stores the sealed secret value,
|
||||
// not the secret *name*, so the manifest's `inbound_secret_ref`
|
||||
// can't be reconstructed — set it up by hand.
|
||||
other => skipped.push(format!("{other} ({})", t.id)),
|
||||
}
|
||||
}
|
||||
for s in &skipped {
|
||||
eprintln!("warning: skipping {s} trigger — not yet representable in the manifest");
|
||||
}
|
||||
|
||||
let manifest = Manifest {
|
||||
app: ManifestApp {
|
||||
slug: app.app.slug.clone(),
|
||||
name: app.app.name.clone(),
|
||||
description: app.app.description.clone(),
|
||||
},
|
||||
scripts: manifest_scripts,
|
||||
routes,
|
||||
triggers: manifest_triggers,
|
||||
secrets: ManifestSecrets {
|
||||
names: secrets.iter().map(|s| s.name.clone()).collect(),
|
||||
},
|
||||
};
|
||||
|
||||
let manifest_path = dir.join(MANIFEST_FILE);
|
||||
std::fs::write(&manifest_path, manifest.to_toml()?)
|
||||
.with_context(|| format!("writing {}", manifest_path.display()))?;
|
||||
|
||||
let mut block = KvBlock::new();
|
||||
block
|
||||
.field("manifest", manifest_path.display().to_string())
|
||||
.field("app", manifest.app.slug.clone())
|
||||
.field("scripts", manifest.scripts.len().to_string())
|
||||
.field("routes", manifest.routes.len().to_string())
|
||||
.field("triggers", trigger_count(&manifest.triggers).to_string())
|
||||
.field("secrets", manifest.secrets.names.len().to_string());
|
||||
block.print(mode);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn trigger_count(t: &ManifestTriggers) -> usize {
|
||||
t.kv.len() + t.docs.len() + t.files.len() + t.cron.len() + t.pubsub.len() + t.queue.len()
|
||||
}
|
||||
|
||||
/// True if `name` is safe to use as a single path component in `scripts/`.
|
||||
/// Rejects empty names, path separators, `.`/`..`, and leading dots.
|
||||
fn is_safe_filename(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& !name.starts_with('.')
|
||||
&& !name.contains('/')
|
||||
&& !name.contains('\\')
|
||||
&& name != ".."
|
||||
&& !name.contains('\0')
|
||||
}
|
||||
|
||||
/// Deserialize a trigger's `details` JSON, attributing failures to the kind.
|
||||
/// The server tags details with a `kind` field which these structs ignore.
|
||||
fn decode_details<T: for<'de> Deserialize<'de>>(
|
||||
details: &serde_json::Value,
|
||||
kind: &str,
|
||||
) -> Result<T> {
|
||||
serde_json::from_value(details.clone())
|
||||
.with_context(|| format!("decoding {kind} trigger details"))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CollectionDetails<Op> {
|
||||
collection_glob: String,
|
||||
#[serde(default = "Vec::new")]
|
||||
ops: Vec<Op>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CronDetails {
|
||||
schedule: String,
|
||||
#[serde(default = "default_timezone")]
|
||||
timezone: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PubsubDetails {
|
||||
topic_pattern: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct QueueDetails {
|
||||
queue_name: String,
|
||||
visibility_timeout_secs: u32,
|
||||
}
|
||||
|
||||
fn default_timezone() -> String {
|
||||
"UTC".to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::is_safe_filename;
|
||||
|
||||
#[test]
|
||||
fn rejects_traversal_and_separators() {
|
||||
for bad in [
|
||||
"",
|
||||
".",
|
||||
"..",
|
||||
"../etc/passwd",
|
||||
"a/b",
|
||||
"a\\b",
|
||||
".hidden",
|
||||
"with\0nul",
|
||||
] {
|
||||
assert!(!is_safe_filename(bad), "expected {bad:?} to be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_normal_names() {
|
||||
for ok in ["create-post", "nightly_digest", "Greet", "x", "a.b"] {
|
||||
assert!(is_safe_filename(ok), "expected {ok:?} to be accepted");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user