Skip to main content

zombienet_orchestrator/network/node/
process.rs

1//! Custom processes: the services a network config declares under
2//! `[[custom_processes]]`, running next to the nodes.
3//!
4//! A [`CustomProcessNode`] is a member of the network like any other node:
5//! it is tracked on [`Network`](crate::network::Network), written to
6//! `zombie.json`, recreated on attach and destroyed with the namespace. It
7//! has no chain, so the only protocol-level surface it exposes is the set of
8//! ports it declared and where each one is reachable.
9
10use std::{collections::BTreeMap, fmt, net::IpAddr, time::Duration};
11
12use async_trait::async_trait;
13use configuration::{types::Port, CustomProcess};
14use provider::types::ProcessStatus;
15use serde::{Deserialize, Serialize};
16use tracing::debug;
17
18use super::{
19    core::NodeCore,
20    spawned::{NodeKind, SpawnedNode},
21};
22
23/// Where one declared port of a custom process is reachable.
24#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
25pub struct ProcessPort {
26    /// The port the process listens on: as declared, or the one picked for `0`.
27    pub port: Port,
28    /// `host:port` as reachable from where zombienet runs: a port-forward in
29    /// k8s (unless running in CI, where it is the pod's own address), the
30    /// published host port on docker, the port itself on native.
31    pub external: String,
32    /// `host:port` as reachable from inside the network (other nodes and
33    /// processes): the process's Service in k8s (`<name>:<port>`), the
34    /// container ip on docker, the same as `external` on native.
35    pub internal: String,
36}
37
38/// Where a custom process is in its lifecycle. In `zombie.json` it is the
39/// `state` field (`starting`, `ready`, `completed`, `failed`) plus
40/// `state_reason` for a failure; a record without one is taken as ready.
41#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
42#[serde(tag = "state", rename_all = "lowercase")]
43pub enum CustomProcessState {
44    /// Spawned, its condition not met yet.
45    Starting,
46    /// Up: passed its ready check, or had nothing to check.
47    Ready,
48    /// A one-shot that exited 0.
49    Completed,
50    /// The ready check failed or timed out, or the one-shot exited with an
51    /// error: what `wait_ready` reports then.
52    Failed {
53        #[serde(rename = "state_reason")]
54        reason: String,
55    },
56}
57
58impl fmt::Display for CustomProcessState {
59    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
60        f.write_str(match self {
61            Self::Starting => "starting",
62            Self::Ready => "ready",
63            Self::Completed => "completed",
64            Self::Failed { .. } => "failed",
65        })
66    }
67}
68
69/// A running custom process.
70#[derive(Clone, Serialize)]
71pub struct CustomProcessNode {
72    #[serde(flatten)]
73    pub(crate) core: NodeCore,
74    pub(crate) spec: CustomProcess,
75    /// Ip the process is reachable at.
76    pub(crate) ip: IpAddr,
77    /// Where each declared port is reachable, by the port's name.
78    pub(crate) ports: BTreeMap<String, ProcessPort>,
79    #[serde(flatten)]
80    pub(crate) state: CustomProcessState,
81}
82
83/// Deserialization counterpart used when re-attaching to a running network.
84#[derive(Deserialize)]
85pub(crate) struct RawCustomProcessNode {
86    pub(crate) name: String,
87    pub(crate) spec: CustomProcess,
88    pub(crate) ip: IpAddr,
89    #[serde(default)]
90    pub(crate) ports: BTreeMap<String, ProcessPort>,
91    /// `None` for a record from before states were kept, or one this version
92    /// cannot read (a state it does not know); taken as up either way.
93    #[serde(flatten)]
94    pub(crate) state: Option<CustomProcessState>,
95    /// The provider's own record of the process, which it needs to reattach.
96    pub(crate) inner: serde_json::Value,
97}
98
99impl CustomProcessNode {
100    pub(crate) fn new(
101        name: impl Into<String>,
102        inner: provider::DynNode,
103        spec: CustomProcess,
104        ip: IpAddr,
105        ports: BTreeMap<String, ProcessPort>,
106        state: CustomProcessState,
107    ) -> Self {
108        Self {
109            core: NodeCore::new(name, inner, NodeKind::CustomProcess),
110            spec,
111            ip,
112            ports,
113            state,
114        }
115    }
116
117    /// The provider-generic part of this process.
118    pub fn core(&self) -> &NodeCore {
119        &self.core
120    }
121
122    pub fn name(&self) -> &str {
123        self.core.name()
124    }
125
126    /// The configuration this process was spawned from.
127    pub fn spec(&self) -> &CustomProcess {
128        &self.spec
129    }
130
131    /// Ip the process is reachable at.
132    pub fn ip(&self) -> IpAddr {
133        self.ip
134    }
135
136    /// Where every declared port is reachable, by the port's name.
137    pub fn ports(&self) -> &BTreeMap<String, ProcessPort> {
138        &self.ports
139    }
140
141    /// Where the port named `name` is reachable, if the process declared it.
142    pub fn port(&self, name: &str) -> Option<&ProcessPort> {
143        self.ports.get(name)
144    }
145
146    /// `host:port` to reach the port named `name` from where zombienet runs,
147    /// if the process declared it.
148    pub fn get_uri_for_name(&self, name: &str) -> Option<&str> {
149        self.ports.get(name).map(|p| p.external.as_str())
150    }
151
152    /// Where the process is in its lifecycle, as of the spawn (or the record
153    /// attached to); [`Self::wait_ready`] re-checks.
154    pub fn state(&self) -> &CustomProcessState {
155        &self.state
156    }
157
158    /// The first declared port, in declaration order, if any.
159    fn first_port(&self) -> Option<&ProcessPort> {
160        self.spec
161            .ports()
162            .first()
163            .and_then(|declared| self.ports.get(&declared.name))
164    }
165
166    /// Wait for the process to meet its condition and say what happened: a
167    /// one-shot to exit (0 is [`CustomProcessState::Completed`]), any other
168    /// to pass its ready check (the provider's own, where it has one, else a
169    /// probe from here). A one-shot that already finished is reported as
170    /// recorded: there is nothing left to observe, and after an attach its
171    /// pid is gone along with its exit code.
172    pub async fn wait_ready(&self, timeout: Duration) -> CustomProcessState {
173        if self.spec.one_shot()
174            && matches!(
175                self.state,
176                CustomProcessState::Completed | CustomProcessState::Failed { .. }
177            )
178        {
179            return self.state.clone();
180        }
181        match self.await_condition(timeout).await {
182            Ok(state) => state,
183            Err(reason) => CustomProcessState::Failed { reason },
184        }
185    }
186
187    async fn await_condition(&self, timeout: Duration) -> Result<CustomProcessState, String> {
188        let interval = Duration::from_secs(u64::from(
189            self.spec.ready_check().map(|c| c.interval).unwrap_or(1),
190        ));
191        let check = self.spec.effective_check();
192        let target = check
193            .as_ref()
194            .and_then(|check| Some((self.ports.get(check.port_name())?, check.http_path())));
195        if !self.spec.one_shot() && target.is_none() {
196            return Ok(CustomProcessState::Ready);
197        }
198        let client = reqwest::Client::builder()
199            .no_proxy()
200            .build()
201            .map_err(|e| format!("could not build the http probe: {e}"))?;
202
203        // The last thing that stood in the way, for the timeout message. A
204        // status read that fails is retried like any other miss: one API
205        // hiccup must not fail the process for good.
206        let mut problem = String::from("no status read yet");
207        let waited = tokio::time::timeout(timeout, async {
208            loop {
209                let status = match self.core.inner().status().await {
210                    Ok(status) => status,
211                    Err(err) => {
212                        problem = format!("could not read the process status: {err}");
213                        tokio::time::sleep(interval).await;
214                        continue;
215                    },
216                };
217                if self.spec.one_shot() {
218                    match status {
219                        ProcessStatus::Exited { code: Some(0) } => {
220                            return Ok(CustomProcessState::Completed)
221                        },
222                        ProcessStatus::Exited { code: Some(code) } => {
223                            return Err(format!("exited with code {code}"))
224                        },
225                        ProcessStatus::Exited { code: None } => {
226                            return Err("exited, with its code unknown".into())
227                        },
228                        ProcessStatus::Running { .. } => problem = "still running".into(),
229                    }
230                } else {
231                    let (port, path) = target.expect("checked above; qed");
232                    let kind = if path.is_some() { "http" } else { "tcp" };
233                    match status {
234                        ProcessStatus::Exited { code } => {
235                            return Err(match code {
236                                Some(code) => format!("exited with code {code}"),
237                                None => "exited".into(),
238                            })
239                        },
240                        // the provider checks (k8s: the pod's probe)
241                        ProcessStatus::Running { ready: Some(true) } => {
242                            return Ok(CustomProcessState::Ready)
243                        },
244                        ProcessStatus::Running { ready: Some(false) } => {
245                            problem = format!(
246                                "the provider's {kind} probe on port {}{} did not pass",
247                                port.port,
248                                path.unwrap_or("")
249                            );
250                        },
251                        // nobody else checks: probe from here
252                        ProcessStatus::Running { ready: None } => {
253                            let passed = match path {
254                                None => tcp_answers(&port.external).await,
255                                Some(path) => http_answers(&client, &port.external, path).await,
256                            };
257                            if passed {
258                                return Ok(CustomProcessState::Ready);
259                            }
260                            problem = format!(
261                                "{kind} {}{} did not pass",
262                                port.external,
263                                path.unwrap_or("")
264                            );
265                        },
266                    }
267                }
268                tokio::time::sleep(interval).await;
269            }
270        })
271        .await;
272
273        match waited {
274            Ok(outcome) => outcome,
275            Err(_elapsed) => Err(format!(
276                "{} after {}s ({problem})",
277                if self.spec.one_shot() {
278                    "not finished"
279                } else {
280                    "not ready"
281                },
282                timeout.as_secs()
283            )),
284        }
285    }
286
287    /// Check if the process is responsive by connecting to its first declared
288    /// port, with a short timeout (2 seconds). A process that declares no port
289    /// has nothing to probe and is reported responsive.
290    ///
291    /// On kubernetes outside CI the address is a local port-forward, and on
292    /// docker a published port behind docker's proxy; both accept a connection
293    /// whether or not the process listens, so there the probe says the forward
294    /// is up, not that the process answers. Native and kubernetes in CI reach
295    /// the process itself.
296    pub async fn is_responsive(&self) -> bool {
297        match self.first_port() {
298            Some(port) => tcp_answers(&port.external).await,
299            None => true,
300        }
301    }
302}
303
304/// One TCP connect to `addr`, given 2 seconds.
305async fn tcp_answers(addr: &str) -> bool {
306    tokio::time::timeout(Duration::from_secs(2), tokio::net::TcpStream::connect(addr))
307        .await
308        .is_ok_and(|connected| connected.is_ok())
309}
310
311/// One `GET http://<addr><path>`, given 5 seconds, passing on 2xx.
312async fn http_answers(client: &reqwest::Client, addr: &str, path: &str) -> bool {
313    client
314        .get(format!("http://{addr}{path}"))
315        .timeout(Duration::from_secs(5))
316        .send()
317        .await
318        .is_ok_and(|res| res.status().is_success())
319}
320
321#[async_trait]
322impl SpawnedNode for CustomProcessNode {
323    fn core(&self) -> &NodeCore {
324        &self.core
325    }
326
327    /// Nothing to wait on here: the spawn already waited for the process's
328    /// condition, and a failed one must not fail a network-wide wait (the
329    /// network is kept up for its logs). [`Self::wait_ready`] re-checks on
330    /// request.
331    async fn wait_until_is_up(&self, _timeout_secs: u64) -> Result<(), anyhow::Error> {
332        debug!(
333            "[{}] custom process, not waited on; state {}",
334            self.name(),
335            self.state
336        );
337        Ok(())
338    }
339}
340
341impl fmt::Debug for CustomProcessNode {
342    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
343        f.debug_struct("CustomProcessNode")
344            .field("inner", &"inner_skipped")
345            .field("name", &self.name())
346            .field("spec", &self.spec)
347            .field("ip", &self.ip)
348            .field("ports", &self.ports)
349            .field("state", &self.state)
350            .finish()
351    }
352}
353
354#[cfg(test)]
355mod tests {
356    use std::sync::Arc;
357
358    use configuration::{CustomProcessBuilder, ReadyCheck};
359
360    use super::*;
361    use crate::network::node::mock::MockNode;
362
363    fn node(spec: CustomProcess, mock: MockNode, state: CustomProcessState) -> CustomProcessNode {
364        let ports = spec
365            .ports()
366            .iter()
367            .map(|p| {
368                (
369                    p.name.clone(),
370                    ProcessPort {
371                        port: 8080,
372                        external: "127.0.0.1:1".into(),
373                        internal: "x:8080".into(),
374                    },
375                )
376            })
377            .collect();
378        CustomProcessNode::new(
379            "p",
380            Arc::new(mock),
381            spec,
382            "127.0.0.1".parse().unwrap(),
383            ports,
384            state,
385        )
386    }
387
388    fn one_shot() -> CustomProcess {
389        CustomProcessBuilder::new()
390            .with_name("p")
391            .with_command("sh")
392            .with_one_shot()
393            .build()
394            .unwrap()
395    }
396
397    fn with_check() -> CustomProcess {
398        CustomProcessBuilder::new()
399            .with_name("p")
400            .with_command("srv")
401            .with_named_port("http", 8080)
402            .with_ready_check(ReadyCheck::tcp("http"))
403            .build()
404            .unwrap()
405    }
406
407    #[tokio::test]
408    async fn wait_ready_follows_what_the_provider_reports() {
409        let running = ProcessStatus::Running { ready: None };
410        let secs = Duration::from_secs;
411
412        // a one-shot: running, then exit 0
413        let n = node(
414            one_shot(),
415            MockNode::new().with_statuses(vec![
416                Ok(running.clone()),
417                Ok(ProcessStatus::Exited { code: Some(0) }),
418            ]),
419            CustomProcessState::Starting,
420        );
421        assert_eq!(n.wait_ready(secs(10)).await, CustomProcessState::Completed);
422
423        // a one-shot that fails
424        let n = node(
425            one_shot(),
426            MockNode::new().with_statuses(vec![Ok(ProcessStatus::Exited { code: Some(3) })]),
427            CustomProcessState::Starting,
428        );
429        assert_eq!(
430            n.wait_ready(secs(10)).await,
431            CustomProcessState::Failed {
432                reason: "exited with code 3".into()
433            }
434        );
435
436        // a finished one-shot is reported as recorded, nothing is read
437        let n = node(
438            one_shot(),
439            MockNode::new().with_statuses(vec![Err("not asked")]),
440            CustomProcessState::Completed,
441        );
442        assert_eq!(n.wait_ready(secs(10)).await, CustomProcessState::Completed);
443
444        // a status error is retried; the provider's probe then passes
445        let n = node(
446            with_check(),
447            MockNode::new().with_statuses(vec![
448                Err("api down"),
449                Ok(ProcessStatus::Running { ready: Some(false) }),
450                Ok(ProcessStatus::Running { ready: Some(true) }),
451            ]),
452            CustomProcessState::Starting,
453        );
454        assert_eq!(n.wait_ready(secs(10)).await, CustomProcessState::Ready);
455
456        // never ready: the timeout names the last problem
457        let n = node(
458            with_check(),
459            MockNode::new().with_statuses(vec![Ok(ProcessStatus::Running { ready: Some(false) })]),
460            CustomProcessState::Starting,
461        );
462        assert_eq!(
463            n.wait_ready(secs(2)).await,
464            CustomProcessState::Failed {
465                reason: "not ready after 2s (the provider's tcp probe on port 8080 did not pass)"
466                    .into()
467            }
468        );
469
470        // exits before becoming ready
471        let n = node(
472            with_check(),
473            MockNode::new().with_statuses(vec![Ok(ProcessStatus::Exited { code: Some(9) })]),
474            CustomProcessState::Starting,
475        );
476        assert_eq!(
477            n.wait_ready(secs(10)).await,
478            CustomProcessState::Failed {
479                reason: "exited with code 9".into()
480            }
481        );
482    }
483
484    #[test]
485    fn the_state_is_two_plain_fields_and_an_old_record_is_ready() {
486        for (state, json) in [
487            (CustomProcessState::Starting, r#"{"state":"starting"}"#),
488            (CustomProcessState::Ready, r#"{"state":"ready"}"#),
489            (CustomProcessState::Completed, r#"{"state":"completed"}"#),
490            (
491                CustomProcessState::Failed {
492                    reason: "exited with code 3".into(),
493                },
494                r#"{"state":"failed","state_reason":"exited with code 3"}"#,
495            ),
496        ] {
497            assert_eq!(serde_json::to_string(&state).unwrap(), json);
498            assert_eq!(
499                serde_json::from_str::<CustomProcessState>(json).unwrap(),
500                state
501            );
502        }
503
504        let with_state = serde_json::json!({
505            "name": "web", "spec": {"name": "web", "command": "web"}, "ip": "10.0.0.1",
506            "state": "failed", "state_reason": "x", "inner": {}
507        });
508        let raw: RawCustomProcessNode = serde_json::from_value(with_state).unwrap();
509        assert_eq!(
510            raw.state,
511            Some(CustomProcessState::Failed { reason: "x".into() })
512        );
513
514        let without_state = serde_json::json!({
515            "name": "web", "spec": {"name": "web", "command": "web"}, "ip": "10.0.0.1", "inner": {}
516        });
517        let raw: RawCustomProcessNode = serde_json::from_value(without_state).unwrap();
518        assert_eq!(raw.state, None);
519    }
520
521    #[test]
522    fn a_raw_process_without_ports_still_deserializes_but_needs_its_inner() {
523        let json = serde_json::json!({
524            "name": "eth-rpc",
525            "spec": { "name": "eth-rpc", "command": "eth-rpc" },
526            "ip": "10.0.0.1",
527            "inner": { "name": "eth-rpc" }
528        });
529        let raw: RawCustomProcessNode = serde_json::from_value(json).unwrap();
530        assert_eq!(raw.name, "eth-rpc");
531        assert!(raw.ports.is_empty());
532
533        let json = serde_json::json!({
534            "name": "eth-rpc",
535            "spec": { "name": "eth-rpc", "command": "eth-rpc" },
536            "ip": "10.0.0.1"
537        });
538        assert!(serde_json::from_value::<RawCustomProcessNode>(json).is_err());
539    }
540
541    #[test]
542    fn ports_round_trip_by_name() {
543        let port = ProcessPort {
544            port: 8545,
545            external: "127.0.0.1:53421".into(),
546            internal: "eth-rpc:8545".into(),
547        };
548        let json = serde_json::to_value(&port).unwrap();
549        assert_eq!(json["external"], "127.0.0.1:53421");
550        let back: ProcessPort = serde_json::from_value(json).unwrap();
551        assert_eq!(back, port);
552    }
553}