Skip to main content

zombienet_orchestrator/
lib.rs

1// TODO(Javier): Remove when we implement the logic in the orchestrator to spawn with the provider.
2#![allow(dead_code, clippy::expect_fun_call)]
3
4pub mod errors;
5pub mod generators;
6pub mod network;
7pub mod network_helper;
8pub mod observability;
9pub mod tx_helper;
10
11mod custom_processes;
12mod network_spec;
13pub mod shared;
14mod spawner;
15mod utils;
16
17use std::{
18    collections::{HashMap, HashSet, VecDeque},
19    env,
20    net::IpAddr,
21    path::{Path, PathBuf},
22    sync::Arc,
23    time::{Duration, SystemTime},
24};
25
26use anyhow::anyhow;
27use configuration::{types::JsonOverrides, NetworkConfig, RegistrationStrategy};
28use errors::OrchestratorError;
29use generators::{core_assignment, errors::GeneratorError, jam_chain_spec};
30use network::{
31    node::{NetworkNode, SpawnedNode},
32    parachain::Parachain,
33    relaychain::Relaychain,
34    Network,
35};
36// re-exported
37pub use network_spec::NetworkSpec;
38use network_spec::{
39    jamchain::{JamChainSpecSource, JamchainSpec},
40    node::NodeSpec,
41    parachain::ParachainSpec,
42};
43use provider::{
44    constants::{HELPERS_DIR, NODE_MOUNTED_DIRS, NODE_SCRIPTS_DIR},
45    shared::helpers::running_in_ci,
46    types::{NodeRole, ProviderCapabilities, TransferedFile},
47    DynNamespace, DynProvider,
48};
49use serde_json::json;
50use support::{
51    constants::{
52        GRAPH_CONTAINS_DEP, GRAPH_CONTAINS_NAME, INDEGREE_CONTAINS_NAME, QUEUE_NOT_EMPTY,
53        THIS_IS_A_BUG,
54    },
55    fs::{FileSystem, FileSystemError},
56    replacer::{get_tokens_to_replace, has_tokens},
57};
58use tokio::time::timeout;
59use tracing::{debug, info, trace, warn};
60
61use crate::{
62    network::{
63        jamchain::{Jamchain, RawJamchain},
64        node::{
65            jam::RawJamNetworkNode, process::RawCustomProcessNode, CustomProcessNode,
66            CustomProcessState, JamNetworkNode, RawNetworkNode,
67        },
68        parachain::RawParachain,
69        relaychain::RawRelaychain,
70    },
71    shared::types::RegisterParachainOptions,
72    spawner::SpawnNodeCtx,
73    utils::write_zombie_json,
74};
75pub struct Orchestrator<T>
76where
77    T: FileSystem + Sync + Send,
78{
79    filesystem: T,
80    provider: DynProvider,
81}
82
83impl<T> Orchestrator<T>
84where
85    T: FileSystem + Sync + Send + Clone,
86{
87    pub fn new(filesystem: T, provider: DynProvider) -> Self {
88        Self {
89            filesystem,
90            provider,
91        }
92    }
93
94    pub async fn spawn(
95        &self,
96        network_config: NetworkConfig,
97    ) -> Result<Network<T>, OrchestratorError> {
98        let global_timeout = network_config.global_settings().network_spawn_timeout();
99        let network_spec = NetworkSpec::from_config(&network_config).await?;
100
101        let res = timeout(
102            Duration::from_secs(global_timeout.into()),
103            self.spawn_inner(network_spec),
104        )
105        .await
106        .map_err(|_| OrchestratorError::GlobalTimeOut(global_timeout));
107        res?
108    }
109
110    pub async fn spawn_from_spec(
111        &self,
112        network_spec: NetworkSpec,
113    ) -> Result<Network<T>, OrchestratorError> {
114        let global_timeout = network_spec.global_settings.network_spawn_timeout();
115        let res = timeout(
116            Duration::from_secs(global_timeout as u64),
117            self.spawn_inner(network_spec),
118        )
119        .await
120        .map_err(|_| OrchestratorError::GlobalTimeOut(global_timeout));
121        res?
122    }
123
124    pub async fn attach_to_live(
125        &self,
126        zombie_json_path: &Path,
127    ) -> Result<Network<T>, OrchestratorError> {
128        info!("attaching to live network...");
129        info!("reading zombie.json from {:?}", zombie_json_path);
130
131        let zombie_json_content = self.filesystem.read_to_string(zombie_json_path).await?;
132        let zombie_json: serde_json::Value = serde_json::from_str(&zombie_json_content)?;
133
134        self.attach_to_live_from_json(&zombie_json).await
135    }
136
137    /// Like [`Self::attach_to_live`], but takes already-parsed `zombie.json` content.
138    /// Its `local_base_dir` is used as-is.
139    pub async fn attach_to_live_from_json(
140        &self,
141        zombie_json: &serde_json::Value,
142    ) -> Result<Network<T>, OrchestratorError> {
143        info!("recreating namespace...");
144        let ns: DynNamespace = self
145            .provider
146            .create_namespace_from_json(zombie_json)
147            .await?;
148
149        info!("recreating relaychain...");
150        let (relay, initial_spec) =
151            recreate_relaychain_from_json(zombie_json, ns.clone(), self.provider.name()).await?;
152        let relay_nodes = relay.nodes.clone();
153
154        let mut network =
155            Network::new_with_relay(relay, ns.clone(), self.filesystem.clone(), initial_spec);
156
157        for node in relay_nodes {
158            if node.is_responsive().await {
159                node.set_is_running(true);
160            }
161            network.insert_node(node);
162        }
163
164        info!("recreating parachains...");
165        let parachains_map =
166            recreate_parachains_from_json(zombie_json, ns.clone(), self.provider.name()).await?;
167        let para_nodes = parachains_map
168            .values()
169            .flat_map(|paras| paras.iter().flat_map(|para| para.collators.clone()))
170            .collect::<Vec<Arc<NetworkNode>>>();
171
172        network.set_parachains(parachains_map);
173        for node in para_nodes {
174            if node.is_responsive().await {
175                node.set_is_running(true);
176            }
177            network.insert_node(node);
178        }
179
180        if let Some(jamchain) =
181            recreate_jamchain_from_json(zombie_json, ns.clone(), self.provider.name()).await?
182        {
183            info!("recreating jamchain...");
184            let jam_nodes = jamchain.nodes.clone();
185            network.set_jamchain(jamchain);
186            for node in jam_nodes {
187                if node.is_responsive().await {
188                    node.core().set_is_running(true);
189                }
190                network.insert_node(node);
191            }
192        }
193
194        let processes =
195            recreate_custom_processes_from_json(zombie_json, ns.clone(), self.provider.name())
196                .await?;
197        if !processes.is_empty() {
198            info!("recreating custom processes...");
199            for process in &processes {
200                // On docker and k8s this reaches a forward or proxy that
201                // answers for a gone process too; a real check is #596.
202                if process.is_responsive().await {
203                    process.core().set_is_running(true);
204                }
205            }
206            network.set_custom_processes(processes);
207        }
208
209        Ok(network)
210    }
211
212    async fn spawn_inner(
213        &self,
214        mut network_spec: NetworkSpec,
215    ) -> Result<Network<T>, OrchestratorError> {
216        // main driver for spawn the network
217        debug!(network_spec = ?network_spec,"Network spec to spawn");
218
219        // TODO: move to Provider trait
220        validate_spec_with_provider_capabilities(&network_spec, self.provider.capabilities())
221            .map_err(|err| {
222                OrchestratorError::InvalidConfigForProvider(
223                    self.provider.name().into(),
224                    err.to_string(),
225                )
226            })?;
227
228        // create namespace
229        let ns = if let Some(base_dir) = network_spec.global_settings.base_dir() {
230            self.provider
231                .create_namespace_with_base_dir(base_dir)
232                .await?
233        } else {
234            self.provider.create_namespace().await?
235        };
236
237        // set the spawn_concurrency
238        let (spawn_concurrency, limited_by_tokens) = calculate_concurrency(&network_spec)?;
239
240        let start_time = SystemTime::now();
241        info!("🧰 ns: {}", ns.name());
242        info!("🧰 base_dir: {:?}", ns.base_dir());
243        info!("🕰 start time: {:?}", start_time);
244        info!("⚙️ spawn concurrency: {spawn_concurrency} (limited by tokens: {limited_by_tokens})");
245
246        network_spec
247            .populate_nodes_available_args(ns.clone())
248            .await?;
249
250        // A JAM chain takes the place of the relaychain and has its own, much simpler,
251        // spawn flow.
252        if network_spec.try_relaychain().is_none() {
253            let jam_spec =
254                network_spec
255                    .jamchain
256                    .clone()
257                    .ok_or(OrchestratorError::InvalidConfig(
258                        "A network needs either a relaychain or a jamchain.".to_string(),
259                    ))?;
260
261            return self
262                .spawn_jam(&jam_spec, &mut network_spec, ns, start_time)
263                .await;
264        }
265
266        // Resolve every node's `db_snapshot` AssetLocation into a local
267        // cache file once, serially, before any parallel spawn.
268        let all_nodes: Vec<&NodeSpec> = network_spec
269            .relaychain()
270            .nodes
271            .iter()
272            .chain(
273                network_spec
274                    .parachains_iter()
275                    .flat_map(|p| p.collators.iter()),
276            )
277            .collect();
278        let resolved_db_snapshots =
279            generators::resolve_db_snapshots(all_nodes, &ns, &self.filesystem).await?;
280
281        let base_dir = ns.base_dir().to_string_lossy();
282        let scoped_fs = ScopedFilesystem::new(&self.filesystem, &base_dir);
283
284        // The relaychain spec is moved out of the network spec while we set it up, so that
285        // it can be mutated independently of the parachains. It's put back before the nodes
286        // are spawned.
287        let mut relaychain = network_spec
288            .relaychain
289            .take()
290            .expect("checked to be `Some` above; qed");
291
292        // Create chain-spec for relaychain
293        relaychain.chain_spec.build(&ns, &scoped_fs).await?;
294
295        debug!("relaychain spec built!");
296        // Create parachain artifacts (chain-spec, wasm, state)
297        let relay_chain_id = relaychain.chain_spec.read_chain_id(&scoped_fs).await?;
298
299        let relay_chain_name = relaychain.chain.as_str().to_owned();
300        let base_dir_exists = network_spec.global_settings.base_dir().is_some();
301        network_spec
302            .build_parachain_artifacts(ns.clone(), &scoped_fs, &relay_chain_id, base_dir_exists)
303            .await?;
304
305        // Gather the parachains to register in genesis and the ones to register with extrinsic
306        let (para_to_register_in_genesis, para_to_register_with_extrinsic): (
307            Vec<&ParachainSpec>,
308            Vec<&ParachainSpec>,
309        ) = network_spec
310            .parachains
311            .iter()
312            .filter(|para| para.registration_strategy != RegistrationStrategy::Manual)
313            .partition(|para| {
314                matches!(para.registration_strategy, RegistrationStrategy::InGenesis)
315            });
316
317        let mut para_artifacts = vec![];
318        for para in &para_to_register_in_genesis {
319            let genesis_config = para.get_genesis_config()?;
320            para_artifacts.push(genesis_config)
321        }
322
323        // Customize relaychain
324        relaychain
325            .chain_spec
326            .customize_relay(
327                &relaychain,
328                &network_spec.hrmp_channels,
329                para_artifacts,
330                &scoped_fs,
331            )
332            .await?;
333
334        // Run post-process script if configured for the relaychain (run against plain spec before building raw)
335        if let Some(script_cmd) = relaychain.post_process_script.as_deref() {
336            relaychain
337                .chain_spec
338                .run_post_process_script(script_cmd, &scoped_fs)
339                .await?;
340        }
341
342        // Override cores if needed
343        let num_cores = network_spec.parachains_iter().fold(0u32, |mut acc, para| {
344            if let Some(cores) = para.num_cores {
345                acc += cores;
346            } else if &RegistrationStrategy::InGenesis == para.registration_strategy() {
347                // add 1 by default
348                acc += 1;
349            }
350
351            acc
352        });
353
354        if num_cores > para_to_register_in_genesis.len() as u32 {
355            let num_cores_to_set = num_cores - para_to_register_in_genesis.len() as u32;
356            // we should set the correct core config
357            let overrides = json!({
358                "configuration": {
359                    "config": {
360                        "scheduler_params": {
361                            "num_cores": num_cores_to_set,
362                            "max_validators_per_core": 1
363                        },
364                    }
365                }
366            });
367
368            relaychain
369                .chain_spec
370                .apply_genesis_override(&scoped_fs, &overrides)
371                .await?;
372        }
373
374        // Build raw version (after any post-processing of the plain spec)
375        relaychain
376            .chain_spec
377            .build_raw(&ns, &scoped_fs, None)
378            .await?;
379
380        // override wasm if needed
381        if let Some(ref wasm_override) = relaychain.wasm_override {
382            relaychain
383                .chain_spec
384                .override_code(&scoped_fs, wasm_override)
385                .await?;
386        }
387
388        // custom override raw spec if needed
389        if let Some(ref raw_spec_override) = relaychain.raw_spec_override {
390            relaychain
391                .chain_spec
392                .override_raw_spec(&scoped_fs, raw_spec_override)
393                .await?;
394        }
395
396        // assign extra cores if needed
397        if num_cores > para_to_register_in_genesis.len() as u32
398            || (relaychain.override_session_0 && !para_to_register_in_genesis.is_empty())
399        {
400            debug!("Raw overrides info: num_cores: {}, para_to_register_in_genesis_len: {:?}, override_session_0: {}", num_cores, para_to_register_in_genesis.len(), relaychain.override_session_0);
401            let mut core_index = 0u32;
402            // we should check with version the runtime is using
403            // could be ParaScheduler or CoretimeAssignmentProvider
404            let scheduler_key = core_assignment::get_parascheduler_storage_key();
405            let is_old = !relaychain
406                .chain_spec
407                .find_raw_key(&scoped_fs, &scheduler_key)
408                .await?;
409
410            let mut para_scheduler_value_parts: Vec<String> = vec![];
411            let mut raw_json_overrides = json!({});
412            // loop over para and assign cores from 0..
413            for para in &network_spec.parachains {
414                // cores we need to assign
415                let mut cores_for_para = 0_u32;
416                if let Some(cores) = para.num_cores {
417                    if &RegistrationStrategy::InGenesis == para.registration_strategy() {
418                        cores_for_para = cores;
419                    }
420                } else {
421                    // no num_cores set but we need to check if `override_session_0` is true
422                    // to assign the first core.
423                    if &RegistrationStrategy::InGenesis == para.registration_strategy()
424                        && relaychain.override_session_0
425                    {
426                        cores_for_para += 1;
427                    }
428                }
429
430                debug!(
431                    "Assigning {cores_for_para} cores in raw spec for para {}. Using pallet {}",
432                    para.id,
433                    if is_old {
434                        "CoretimeAssignmentProvider"
435                    } else {
436                        "ParaScheduler"
437                    }
438                );
439                for _core in 0..cores_for_para {
440                    if is_old {
441                        let (core_assign_key, core_assign_value) =
442                            core_assignment::generate_old(core_index, para.id);
443                        raw_json_overrides[core_assign_key] = json!(core_assign_value);
444                    } else {
445                        let part = core_assignment::generate(core_index, para.id);
446                        para_scheduler_value_parts.push(part);
447                    }
448                    core_index += 1;
449                }
450            }
451
452            // if not old we need to store the k/v to override
453            if !is_old {
454                let count_prefix = format!("{:02x}", para_scheduler_value_parts.len() * 4);
455                let core_assign_value =
456                    format!("{count_prefix}{}", para_scheduler_value_parts.join(""));
457                raw_json_overrides[scheduler_key] = json!(core_assign_value);
458            }
459
460            // extra check to ensure we need to override session 0
461            if relaychain.override_session_0 {
462                trace!("Overriding pallet ParaSessionInfo.session (0) to allow paras to produce blocks at first session.");
463                let raw_spec = relaychain.chain_spec.read_raw_spec(&scoped_fs).await?;
464                let overrides = generators::generate_session_0_overrides(&raw_spec, num_cores)?;
465
466                for (k, v) in overrides.as_object().ok_or(anyhow!(
467                    "'generate_session_0_overrides' should be a valid json Object."
468                ))? {
469                    raw_json_overrides[k] = v.clone();
470                }
471            }
472
473            debug!("Raw overrides keys: {:?}", raw_json_overrides);
474
475            relaychain
476                .chain_spec
477                .override_raw_spec(
478                    &scoped_fs,
479                    &JsonOverrides::Json(json!({
480                        "genesis": {
481                            "raw": {
482                                "top": raw_json_overrides
483                            }
484                        }
485                    })),
486                )
487                .await?;
488        }
489
490        // The relaychain is fully set up now, put it back into the network spec.
491        network_spec.relaychain = Some(relaychain);
492
493        let (bootnodes, relaynodes) =
494            split_nodes_by_bootnodes(&network_spec.relaychain().nodes, false);
495
496        // TODO: we want to still supporting spawn a dedicated bootnode??
497        let mut ctx = SpawnNodeCtx {
498            chain_id: &relay_chain_id,
499            parachain_id: None,
500            chain: relay_chain_name.as_str(),
501            role: ZombieRole::Node,
502            ns: &ns,
503            scoped_fs: &scoped_fs,
504            parachain: None,
505            bootnodes_addr: &vec![],
506            wait_ready: false,
507            nodes_by_name: json!({}),
508            global_settings: &network_spec.global_settings,
509            resolved_db_snapshots: &resolved_db_snapshots,
510        };
511
512        let global_files_to_inject = vec![TransferedFile::new(
513            PathBuf::from(format!(
514                "{}/{relay_chain_name}.json",
515                ns.base_dir().to_string_lossy()
516            )),
517            PathBuf::from(format!("/cfg/{relay_chain_name}.json")),
518        )];
519
520        let r = Relaychain::new(
521            relay_chain_name.to_string(),
522            relay_chain_id.clone(),
523            PathBuf::from(network_spec.relaychain().chain_spec.raw_path().ok_or(
524                OrchestratorError::InvariantError("chain-spec raw path should be set now"),
525            )?),
526        );
527        let mut network =
528            Network::new_with_relay(r, ns.clone(), self.filesystem.clone(), network_spec.clone());
529
530        // Initiate the node_ws_url which will be later used in the Parachain_with_extrinsic config
531        let mut node_ws_url: String = "".to_string();
532
533        // Calculate the bootnodes addr from the running nodes
534        let mut bootnodes_addr: Vec<String> = vec![];
535
536        for level in dependency_levels_among(&bootnodes)? {
537            let mut running_nodes_per_level = vec![];
538            for chunk in level.chunks(spawn_concurrency) {
539                let spawning_tasks = chunk
540                    .iter()
541                    .map(|node| spawner::spawn_node(node, global_files_to_inject.clone(), &ctx));
542
543                for node in futures::future::try_join_all(spawning_tasks).await? {
544                    let bootnode_multiaddr = node.multiaddr();
545
546                    bootnodes_addr.push(bootnode_multiaddr.to_string());
547
548                    // Is used in the register_para_options (We need to get this from the relay and not the collators)
549                    if node_ws_url.is_empty() {
550                        node_ws_url.clone_from(&node.ws_uri)
551                    }
552
553                    running_nodes_per_level.push(node);
554                }
555            }
556            info!(
557                "🕰  waiting for level: {:?} to be up...",
558                level.iter().map(|n| n.name.clone()).collect::<Vec<_>>()
559            );
560
561            // Wait for all nodes in the current level to be up
562            let waiting_tasks = running_nodes_per_level.iter().map(|node| {
563                node.wait_until_is_up(network_spec.global_settings.node_spawn_timeout())
564            });
565
566            let _ = futures::future::try_join_all(waiting_tasks).await?;
567
568            for node in running_nodes_per_level {
569                // Add the node to the  context and `Network` instance
570                ctx.nodes_by_name[node.name().to_owned()] = serde_json::to_value(&node)?;
571                network.add_running_node(node, None).await;
572            }
573        }
574
575        // Add the bootnodes to the relaychain spec file and ctx
576        network_spec
577            .relaychain()
578            .chain_spec
579            .add_bootnodes(&scoped_fs, &bootnodes_addr)
580            .await?;
581
582        ctx.bootnodes_addr = &bootnodes_addr;
583
584        // Upload the raw relaychain spec (with bootnodes) to the provider's file server, if any
585        upload_raw_spec(&ns, &network_spec.relaychain().chain_spec).await?;
586
587        for level in dependency_levels_among(&relaynodes)? {
588            let mut running_nodes_per_level = vec![];
589            for chunk in level.chunks(spawn_concurrency) {
590                let spawning_tasks = chunk
591                    .iter()
592                    .map(|node| spawner::spawn_node(node, global_files_to_inject.clone(), &ctx));
593
594                for node in futures::future::try_join_all(spawning_tasks).await? {
595                    running_nodes_per_level.push(node);
596                }
597            }
598            info!(
599                "🕰 waiting for level: {:?} to be up...",
600                level.iter().map(|n| n.name.clone()).collect::<Vec<_>>()
601            );
602
603            // Wait for all nodes in the current level to be up
604            let waiting_tasks = running_nodes_per_level.iter().map(|node| {
605                node.wait_until_is_up(network_spec.global_settings.network_spawn_timeout())
606            });
607
608            let _ = futures::future::try_join_all(waiting_tasks).await?;
609
610            for node in running_nodes_per_level {
611                ctx.nodes_by_name[node.name().to_owned()] = serde_json::to_value(&node)?;
612                network.add_running_node(node, None).await;
613            }
614        }
615
616        // spawn paras
617        self.spawn_parachains(
618            &network_spec.parachains,
619            &ctx,
620            &global_files_to_inject,
621            spawn_concurrency,
622            &mut network,
623        )
624        .await?;
625
626        // Now we need to register the paras with extrinsic from the Vec collected before;
627        for para in para_to_register_with_extrinsic {
628            let register_para_options: RegisterParachainOptions = RegisterParachainOptions {
629                id: para.id,
630                // This needs to resolve correctly
631                wasm_path: para
632                    .genesis_wasm
633                    .artifact_path()
634                    .ok_or(OrchestratorError::InvariantError(
635                        "artifact path for wasm must be set at this point",
636                    ))?
637                    .to_path_buf(),
638                state_path: para
639                    .genesis_state
640                    .artifact_path()
641                    .ok_or(OrchestratorError::InvariantError(
642                        "artifact path for state must be set at this point",
643                    ))?
644                    .to_path_buf(),
645                node_ws_url: node_ws_url.clone(),
646                onboard_as_para: para.onboard_as_parachain,
647                seed: None, // TODO: Seed is passed by?
648                finalization: false,
649            };
650
651            Parachain::register(register_para_options, &scoped_fs).await?;
652        }
653
654        custom_processes::spawn_custom_processes(
655            &mut network,
656            &network_spec,
657            ns.clone(),
658            configured_concurrency(&network_spec),
659        )
660        .await?;
661
662        // After the processes: a failed one fails the spawn, and these
663        // containers are not torn down with the namespace.
664        if network_spec.global_settings.observability().enabled() {
665            match network
666                .start_observability(network_spec.global_settings.observability())
667                .await
668            {
669                Ok(obs) => {
670                    info!("📊 Prometheus URL: {}", obs.prometheus_url);
671                    info!("📊 Grafana URL: {}", obs.grafana_url);
672                },
673                Err(e) => {
674                    warn!("⚠️  Failed to spawn observability stack: {e}");
675                },
676            }
677        }
678
679        network.set_start_time_ts(start_time);
680
681        write_zombie_json(serde_json::to_value(&network)?, scoped_fs, ns.name()).await?;
682
683        if network_spec.global_settings.tear_down_on_failure() {
684            network.spawn_watching_task();
685        }
686
687        generators::cleanup_db_snapshot_cache(&resolved_db_snapshots).await;
688
689        Ok(network)
690    }
691
692    /// Spawn the collators of every parachain, in the context of an already running network.
693    ///
694    /// `ctx` is the context of the chain the parachains are anchored to (relaychain or JAM
695    /// chain); the per-parachain context is derived from it.
696    async fn spawn_parachains<'a>(
697        &self,
698        parachains: &[ParachainSpec],
699        ctx: &SpawnNodeCtx<'a, T>,
700        files_to_inject: &[TransferedFile],
701        spawn_concurrency: usize,
702        network: &mut Network<T>,
703    ) -> Result<(), OrchestratorError> {
704        let scoped_fs = ctx.scoped_fs;
705
706        for para in parachains {
707            // Create parachain (in the context of the running network)
708            let parachain = Parachain::from_spec(para, files_to_inject, scoped_fs).await?;
709            let parachain_id = parachain.chain_id.clone();
710
711            let (bootnodes, collators) =
712                split_nodes_by_bootnodes(&para.collators, para.no_default_bootnodes);
713
714            // Create `ctx` for spawn parachain nodes
715            let mut ctx_para = SpawnNodeCtx {
716                parachain: Some(para),
717                parachain_id: parachain_id.as_deref(),
718                role: if para.is_cumulus_based {
719                    ZombieRole::CumulusCollator
720                } else {
721                    ZombieRole::Collator
722                },
723                bootnodes_addr: &vec![],
724                ..ctx.clone()
725            };
726
727            // Calculate the bootnodes addr from the running nodes
728            let mut bootnodes_addr: Vec<String> = vec![];
729            let mut running_nodes: Vec<NetworkNode> = vec![];
730
731            for level in dependency_levels_among(&bootnodes)? {
732                for node in self
733                    .spawn_parachain_level(&level, &parachain, &ctx_para, spawn_concurrency)
734                    .await?
735                {
736                    bootnodes_addr.push(node.multiaddr().to_string());
737                    ctx_para.nodes_by_name[node.name().to_owned()] = serde_json::to_value(&node)?;
738                    running_nodes.push(node);
739                }
740            }
741
742            if let Some(para_chain_spec) = para.chain_spec.as_ref() {
743                para_chain_spec
744                    .add_bootnodes(scoped_fs, &bootnodes_addr)
745                    .await?;
746
747                // Upload the raw parachain spec (with bootnodes) to the provider's file server, if any
748                upload_raw_spec(ctx.ns, para_chain_spec).await?;
749            }
750
751            ctx_para.bootnodes_addr = &bootnodes_addr;
752
753            // Spawn the rest of the nodes
754            for level in dependency_levels_among(&collators)? {
755                for node in self
756                    .spawn_parachain_level(&level, &parachain, &ctx_para, spawn_concurrency)
757                    .await?
758                {
759                    ctx_para.nodes_by_name[node.name().to_owned()] = serde_json::to_value(&node)?;
760                    running_nodes.push(node);
761                }
762            }
763
764            let running_para_id = parachain.para_id;
765            network.add_para(parachain);
766            for node in running_nodes {
767                network.add_running_node(node, Some(running_para_id)).await;
768            }
769        }
770
771        Ok(())
772    }
773
774    /// Spawn one dependency level of collators, concurrently, and wait until they are up.
775    async fn spawn_parachain_level<'a>(
776        &self,
777        level: &[&NodeSpec],
778        parachain: &Parachain,
779        ctx: &SpawnNodeCtx<'a, T>,
780        spawn_concurrency: usize,
781    ) -> Result<Vec<NetworkNode>, OrchestratorError> {
782        let mut running_nodes = vec![];
783        for chunk in level.chunks(spawn_concurrency) {
784            let spawning_tasks = chunk
785                .iter()
786                .map(|node| spawner::spawn_node(node, parachain.files_to_inject.clone(), ctx));
787
788            running_nodes.extend(futures::future::try_join_all(spawning_tasks).await?);
789        }
790
791        info!(
792            "🕰  waiting for level: {:?} to be up...",
793            level.iter().map(|n| n.name.clone()).collect::<Vec<_>>()
794        );
795
796        // Wait for all nodes in the current level to be up
797        let waiting_tasks = running_nodes
798            .iter()
799            .map(|node| node.wait_until_is_up(ctx.global_settings.network_spawn_timeout()));
800
801        let _ = futures::future::try_join_all(waiting_tasks).await?;
802
803        Ok(running_nodes)
804    }
805
806    /// Spawn a network whose root chain is a JAM chain instead of a relaychain.
807    async fn spawn_jam(
808        &self,
809        jam_spec: &JamchainSpec,
810        network_spec: &mut NetworkSpec,
811        ns: DynNamespace,
812        start_time: SystemTime,
813    ) -> Result<Network<T>, OrchestratorError> {
814        let base_dir = ns.base_dir().to_string_lossy().to_string();
815        let scoped_fs = ScopedFilesystem::new(&self.filesystem, &base_dir);
816
817        let jam_spec_full_path = format!("{}/{}", base_dir, "jam_spec.json");
818        match &jam_spec.chain_spec_source {
819            JamChainSpecSource::Path(location) => {
820                info!("📜 using JAM chain spec {location}");
821                let spec = location.get_asset().await.map_err(|e| {
822                    GeneratorError::ChainSpecGeneration(format!("JAM chain spec {location}: {e}"))
823                })?;
824                scoped_fs.write("jam_spec.json", spec).await?;
825            },
826            JamChainSpecSource::JamBuilder(genesis) => {
827                info!("🛠️ building the JAM chain spec in process");
828                let built = jam_chain_spec::build(jam_spec, genesis).await?;
829                scoped_fs.write("jam_spec.json", built.spec).await?;
830                scoped_fs
831                    .write(
832                        "jam_genesis.json",
833                        serde_json::to_string_pretty(&built.summary)?,
834                    )
835                    .await?;
836            },
837        }
838
839        // Create parachain artifacts (chain-spec, wasm, state). The parachains are anchored
840        // to the JAM chain, so its id is the one written into their chain-spec.
841        let base_dir_exists = network_spec.global_settings.base_dir().is_some();
842        network_spec
843            .build_parachain_artifacts(
844                ns.clone(),
845                &scoped_fs,
846                jam_spec.id.as_str(),
847                base_dir_exists,
848            )
849            .await?;
850
851        // Resolve every collator's `db_snapshot` AssetLocation into a local cache file once,
852        // serially, before any parallel spawn.
853        let collators: Vec<&NodeSpec> = network_spec
854            .parachains_iter()
855            .flat_map(|para| para.collators.iter())
856            .collect();
857        let resolved_db_snapshots =
858            generators::resolve_db_snapshots(collators, &ns, &self.filesystem).await?;
859
860        let (spawn_concurrency, _) = calculate_concurrency(network_spec)?;
861        let global_settings = &network_spec.global_settings;
862
863        // context setup
864        let jam_ctx = SpawnNodeCtx {
865            chain_id: jam_spec.id.as_str(),
866            parachain_id: None,
867            chain: jam_spec.id.as_str(),
868            role: ZombieRole::Node,
869            ns: &ns,
870            scoped_fs: &scoped_fs,
871            parachain: None,
872            bootnodes_addr: &vec![],
873            wait_ready: false,
874            nodes_by_name: json!({}),
875            global_settings,
876            resolved_db_snapshots: &resolved_db_snapshots,
877        };
878
879        let jam_global_files_to_inject = vec![TransferedFile::new(
880            PathBuf::from(format!("{base_dir}/jam_spec.json")),
881            PathBuf::from("/cfg/jam_spec.json"),
882        )];
883
884        let mut network =
885            Network::new_without_relay(ns.clone(), self.filesystem.clone(), network_spec.clone());
886
887        network.set_jamchain(Jamchain::new(
888            jam_spec.id.as_str(),
889            PathBuf::from(&jam_spec_full_path),
890        ));
891
892        let mut bootnodes_addr = vec![];
893        // Nodes spawned so far, so the args of the next ones can reference them.
894        let mut nodes_by_name = json!({});
895        for jam_node in &jam_spec.nodes {
896            let jam_ctx = SpawnNodeCtx {
897                bootnodes_addr: &bootnodes_addr,
898                nodes_by_name: nodes_by_name.clone(),
899                ..jam_ctx.clone()
900            };
901
902            let running_node =
903                spawner::spawn_jam_node(jam_node, jam_global_files_to_inject.clone(), &jam_ctx)
904                    .await?;
905
906            running_node
907                .wait_until_is_up(global_settings.node_spawn_timeout() as u64)
908                .await
909                .map_err(|e| OrchestratorError::InvalidConfig(e.to_string()))?;
910
911            bootnodes_addr.push(running_node.peer_addr().to_string());
912            nodes_by_name[running_node.name().to_owned()] = serde_json::to_value(&running_node)?;
913            network.add_running_jam_node(running_node).await;
914        }
915
916        // spawn paras
917        for para in network_spec.parachains.iter() {
918            if para.registration_strategy() != &RegistrationStrategy::Manual {
919                warn!(
920                    "⚠️  Parachain {} can't be registered automatically on a JAM chain, it needs to be registered manually.",
921                    para.id
922                );
923            }
924        }
925
926        // the JAM nodes are already running, so the collators can reference them
927        let jam_ctx = SpawnNodeCtx {
928            nodes_by_name,
929            ..jam_ctx
930        };
931
932        self.spawn_parachains(
933            &network_spec.parachains,
934            &jam_ctx,
935            &jam_global_files_to_inject,
936            spawn_concurrency,
937            &mut network,
938        )
939        .await?;
940
941        custom_processes::spawn_custom_processes(
942            &mut network,
943            network_spec,
944            ns.clone(),
945            configured_concurrency(network_spec),
946        )
947        .await?;
948
949        network.set_start_time_ts(start_time);
950
951        write_zombie_json(serde_json::to_value(&network)?, scoped_fs, ns.name()).await?;
952
953        if network_spec.global_settings.tear_down_on_failure() {
954            network.spawn_watching_task();
955        }
956
957        generators::cleanup_db_snapshot_cache(&resolved_db_snapshots).await;
958
959        Ok(network)
960    }
961}
962
963// Helpers
964
965/// Make sure a node persisted in `zombie.json` was spawned by the provider we
966/// are attaching with.
967fn validate_provider_tag(
968    inner: &serde_json::Value,
969    node_name: &str,
970    provider_name: &str,
971) -> Result<(), OrchestratorError> {
972    let provider_tag = inner
973        .get("provider_tag")
974        .and_then(|v| v.as_str())
975        .ok_or_else(|| {
976            OrchestratorError::InvalidConfig(format!(
977                "Node '{node_name}' is missing `provider_tag` in inner node JSON"
978            ))
979        })?;
980
981    if provider_tag != provider_name {
982        return Err(OrchestratorError::InvalidConfigForProvider(
983            provider_name.to_string(),
984            provider_tag.to_string(),
985        ));
986    }
987
988    Ok(())
989}
990
991async fn recreate_network_nodes_from_json(
992    nodes_json: &serde_json::Value,
993    ns: DynNamespace,
994    provider_name: &str,
995) -> Result<Vec<Arc<NetworkNode>>, OrchestratorError> {
996    let raw_nodes: Vec<RawNetworkNode> = serde_json::from_value(nodes_json.clone())?;
997
998    let mut nodes = Vec::with_capacity(raw_nodes.len());
999    for raw in raw_nodes {
1000        validate_provider_tag(&raw.inner, &raw.name, provider_name)?;
1001
1002        let inner = ns.spawn_node_from_json(&raw.inner).await?;
1003        let relay_node = NetworkNode::new(
1004            raw.name,
1005            raw.ws_uri,
1006            raw.internal_ws_uri,
1007            raw.prometheus_uri,
1008            raw.multiaddr,
1009            raw.spec,
1010            inner,
1011            raw.cmd_generator_opts,
1012            raw.context,
1013        );
1014        nodes.push(Arc::new(relay_node));
1015    }
1016
1017    Ok(nodes)
1018}
1019
1020async fn recreate_relaychain_from_json(
1021    zombie_json: &serde_json::Value,
1022    ns: DynNamespace,
1023    provider_name: &str,
1024) -> Result<(Relaychain, NetworkSpec), OrchestratorError> {
1025    let relay_json = zombie_json
1026        .get("relay")
1027        .ok_or(OrchestratorError::InvalidConfig(
1028            "Missing `relay` field in zombie.json".into(),
1029        ))?
1030        .clone();
1031
1032    let mut relay_raw: RawRelaychain = serde_json::from_value(relay_json)?;
1033
1034    let initial_spec: NetworkSpec = serde_json::from_value(
1035        zombie_json
1036            .get("initial_spec")
1037            .ok_or(OrchestratorError::InvalidConfig(
1038                "Missing `initial_spec` field in zombie.json".into(),
1039            ))?
1040            .clone(),
1041    )?;
1042
1043    // Populate relay nodes
1044    let nodes =
1045        recreate_network_nodes_from_json(&relay_raw.nodes, ns.clone(), provider_name).await?;
1046    relay_raw.inner.nodes = nodes;
1047
1048    Ok((relay_raw.inner, initial_spec))
1049}
1050
1051/// Rebuild the `jamchain` section of a `zombie.json`, if the network had one.
1052async fn recreate_jamchain_from_json(
1053    zombie_json: &serde_json::Value,
1054    ns: DynNamespace,
1055    provider_name: &str,
1056) -> Result<Option<Jamchain>, OrchestratorError> {
1057    let Some(jamchain_json) = zombie_json.get("jamchain") else {
1058        return Ok(None);
1059    };
1060
1061    let mut jamchain_raw: RawJamchain = serde_json::from_value(jamchain_json.clone())?;
1062    let raw_nodes: Vec<RawJamNetworkNode> = serde_json::from_value(jamchain_raw.nodes.clone())?;
1063
1064    let mut nodes = Vec::with_capacity(raw_nodes.len());
1065    for raw in raw_nodes {
1066        validate_provider_tag(&raw.inner, &raw.name, provider_name)?;
1067
1068        let inner = ns.spawn_node_from_json(&raw.inner).await?;
1069        nodes.push(Arc::new(JamNetworkNode::new(
1070            raw.name,
1071            inner,
1072            raw.spec,
1073            raw.ip,
1074            raw.cmd_generator_opts,
1075        )));
1076    }
1077
1078    jamchain_raw.inner.nodes = nodes;
1079
1080    Ok(Some(jamchain_raw.inner))
1081}
1082
1083/// Rebuild the `custom_processes` section of a `zombie.json`; empty when the
1084/// network had none (or predates custom processes being tracked).
1085async fn recreate_custom_processes_from_json(
1086    zombie_json: &serde_json::Value,
1087    ns: DynNamespace,
1088    provider_name: &str,
1089) -> Result<Vec<Arc<CustomProcessNode>>, OrchestratorError> {
1090    let Some(processes_json) = zombie_json.get("custom_processes") else {
1091        return Ok(vec![]);
1092    };
1093
1094    let raw_processes: Vec<RawCustomProcessNode> = serde_json::from_value(processes_json.clone())?;
1095
1096    let mut processes = Vec::with_capacity(raw_processes.len());
1097    for raw in raw_processes {
1098        validate_provider_tag(&raw.inner, &raw.name, provider_name)?;
1099
1100        // Attached as recorded, like a node, with the state it had; `state()`
1101        // and `wait_ready()` say more afterwards. A record the provider cannot
1102        // take back is skipped with a warning, the rest of the network attaches.
1103        let inner = match ns.spawn_node_from_json(&raw.inner).await {
1104            Ok(inner) => inner,
1105            Err(err) => {
1106                warn!(
1107                    "⚠️  {}: could not attach to the custom process, skipping it: {err}",
1108                    raw.name
1109                );
1110                continue;
1111            },
1112        };
1113
1114        // The `external` addresses saved on k8s were port-forwards of the
1115        // process that spawned the network, gone with it. Open new ones, and
1116        // keep the saved address where there is none to open.
1117        let mut ports = raw.ports;
1118        let state = raw.state.unwrap_or(CustomProcessState::Ready);
1119        // a one-shot is done by now, nothing to forward to; anything else may
1120        // still be up, whatever its recorded state
1121        if !running_in_ci() && !raw.spec.one_shot() {
1122            for (port_name, port) in ports.iter_mut() {
1123                let saved = std::mem::take(&mut port.external);
1124                port.external =
1125                    spawner::forward_or_keep(&inner, &raw.name, port_name, port.port, saved).await;
1126            }
1127        }
1128
1129        processes.push(Arc::new(CustomProcessNode::new(
1130            raw.name, inner, raw.spec, raw.ip, ports, state,
1131        )));
1132    }
1133
1134    Ok(processes)
1135}
1136
1137async fn recreate_parachains_from_json(
1138    zombie_json: &serde_json::Value,
1139    ns: DynNamespace,
1140    provider_name: &str,
1141) -> Result<HashMap<u32, Vec<Parachain>>, OrchestratorError> {
1142    let paras_json = zombie_json
1143        .get("parachains")
1144        .ok_or(OrchestratorError::InvalidConfig(
1145            "Missing `parachains` field in zombie.json".into(),
1146        ))?
1147        .clone();
1148
1149    let raw_paras: HashMap<u32, Vec<RawParachain>> = serde_json::from_value(paras_json)?;
1150
1151    let mut parachains_map = HashMap::new();
1152
1153    for (id, parachain_entries) in raw_paras {
1154        let mut parsed_vec = Vec::with_capacity(parachain_entries.len());
1155
1156        for raw_para in parachain_entries {
1157            let mut para = raw_para.inner;
1158            para.collators =
1159                recreate_network_nodes_from_json(&raw_para.collators, ns.clone(), provider_name)
1160                    .await?;
1161            parsed_vec.push(para);
1162        }
1163
1164        parachains_map.insert(id, parsed_vec);
1165    }
1166
1167    Ok(parachains_map)
1168}
1169
1170// Split the node list depending if it's bootnode or not
1171// NOTE: if there isn't a bootnode declared we use the first one
1172fn split_nodes_by_bootnodes(
1173    nodes: &[NodeSpec],
1174    no_default_bootnodes: bool,
1175) -> (Vec<&NodeSpec>, Vec<&NodeSpec>) {
1176    // get the bootnodes to spawn first and calculate the bootnode string for use later
1177    let mut bootnodes = vec![];
1178    let mut other_nodes = vec![];
1179    nodes.iter().for_each(|node| {
1180        if node.is_bootnode {
1181            bootnodes.push(node)
1182        } else {
1183            other_nodes.push(node)
1184        }
1185    });
1186
1187    if bootnodes.is_empty() && !no_default_bootnodes {
1188        bootnodes.push(other_nodes.remove(0))
1189    }
1190
1191    (bootnodes, other_nodes)
1192}
1193
1194// Generate a bootnode multiaddress and return as string
1195fn generate_bootnode_addr(
1196    node: &NetworkNode,
1197    ip: &IpAddr,
1198    port: u16,
1199) -> Result<String, GeneratorError> {
1200    generators::generate_node_bootnode_addr(
1201        &node.spec.peer_id,
1202        ip,
1203        port,
1204        node.args().as_ref(),
1205        &node.spec.p2p_cert_hash,
1206    )
1207}
1208
1209// Upload the raw chain-spec to the provider's file server (a no-op for providers without one),
1210// so the nodes spawned after it find the spec already there instead of each uploading it.
1211// The path is the one the nodes inject (`<base_dir>/<chain_spec_name>.json`), so their upload
1212// resolves to the same `<hash>__<file name>` entry.
1213async fn upload_raw_spec(
1214    ns: &DynNamespace,
1215    chain_spec: &generators::chain_spec::ChainSpec,
1216) -> Result<(), OrchestratorError> {
1217    if let Some(location) = chain_spec.raw_path() {
1218        ns.upload_to_fileserver(&ns.base_dir().join(location))
1219            .await?;
1220    }
1221
1222    Ok(())
1223}
1224
1225// Validate that the config fulfill all the requirements of the provider
1226fn validate_spec_with_provider_capabilities(
1227    network_spec: &NetworkSpec,
1228    capabilities: &ProviderCapabilities,
1229) -> Result<(), anyhow::Error> {
1230    let mut errs: Vec<String> = vec![];
1231
1232    for process in &network_spec.custom_processes {
1233        for var in process.env() {
1234            if !capabilities.has_secrets
1235                && matches!(var.value, configuration::EnvValue::Secret { .. })
1236            {
1237                errs.push(format!(
1238                    "custom_processes['{}'].env: '{}' is read from a Secret, which only kubernetes has",
1239                    process.name(),
1240                    var.name
1241                ));
1242            }
1243        }
1244        for file in process.injected_files() {
1245            if NODE_MOUNTED_DIRS
1246                .iter()
1247                .any(|dir| Path::new(dir) == file.remote_path)
1248            {
1249                errs.push(format!(
1250                    "custom_processes['{}'].injected_files: '{}' is a directory zombienet mounts",
1251                    process.name(),
1252                    file.remote_path.display()
1253                ));
1254            } else if let Some(shared) = [NODE_SCRIPTS_DIR, HELPERS_DIR]
1255                .iter()
1256                .find(|dir| file.remote_path.starts_with(dir))
1257            {
1258                // on docker these two are volumes every container of the network shares
1259                errs.push(format!(
1260                    "custom_processes['{}'].injected_files: '{}' is inside {shared}, which the containers of a network share",
1261                    process.name(),
1262                    file.remote_path.display()
1263                ));
1264            }
1265            if !file.local_path.is_file() {
1266                let from = if file.local_path.is_relative() {
1267                    std::env::current_dir()
1268                        .map(|cwd| format!(" (relative to {})", cwd.display()))
1269                        .unwrap_or_default()
1270                } else {
1271                    String::new()
1272                };
1273                errs.push(format!(
1274                    "custom_processes['{}'].injected_files: no file at '{}'{from}",
1275                    process.name(),
1276                    file.local_path.display()
1277                ));
1278            }
1279        }
1280    }
1281
1282    if capabilities.requires_image {
1283        // Relaychain
1284        if let Some(relaychain) = network_spec.try_relaychain() {
1285            if relaychain.default_image.is_none() {
1286                // we should check if each node have an image
1287                let nodes = &relaychain.nodes;
1288                if nodes.iter().any(|node| node.image.is_none()) {
1289                    errs.push(String::from(
1290                        "Missing image for node, and not default is set at relaychain",
1291                    ));
1292                }
1293            }
1294        };
1295
1296        // Paras
1297        for para in &network_spec.parachains {
1298            if para.default_image.is_none() {
1299                let nodes = &para.collators;
1300                if nodes.iter().any(|node| node.image.is_none()) {
1301                    errs.push(format!(
1302                        "Missing image for node, and not default is set at parachain {}",
1303                        para.id
1304                    ));
1305                }
1306            }
1307        }
1308    } else {
1309        // native
1310        // We need to get all the `cmds` and verify if are part of the path
1311        let mut cmds: HashSet<&str> = Default::default();
1312        if let Some(relaychain) = network_spec.try_relaychain() {
1313            if let Some(cmd) = relaychain.default_command.as_ref() {
1314                cmds.insert(cmd.as_str());
1315            }
1316            for node in relaychain.nodes.iter() {
1317                cmds.insert(node.command());
1318            }
1319        }
1320
1321        if let Some(jamchain) = network_spec.jamchain.as_ref() {
1322            for node in jamchain.nodes.iter() {
1323                cmds.insert(node.command());
1324            }
1325        }
1326
1327        // Paras
1328        for para in &network_spec.parachains {
1329            if let Some(cmd) = para.default_command.as_ref() {
1330                cmds.insert(cmd.as_str());
1331            }
1332
1333            for node in para.collators.iter() {
1334                cmds.insert(node.command());
1335            }
1336        }
1337
1338        // now check the binaries
1339        let path = std::env::var("PATH").unwrap_or_default(); // path should always be set
1340        trace!("current PATH: {path}");
1341        let parts: Vec<_> = path.split(":").collect();
1342        for cmd in cmds {
1343            let missing = if cmd.contains('/') {
1344                trace!("checking {cmd}");
1345                if std::fs::metadata(cmd).is_err() {
1346                    true
1347                } else {
1348                    info!("🔎  We will use the full path {cmd} to spawn nodes.");
1349                    false
1350                }
1351            } else {
1352                // should be in the PATH
1353                !parts.iter().any(|part| {
1354                    let path_to = format!("{part}/{cmd}");
1355                    trace!("checking {path_to}");
1356                    let check_result = std::fs::metadata(&path_to);
1357                    trace!("result {:?}", check_result);
1358                    if check_result.is_ok() {
1359                        info!("🔎  We will use the cmd: '{cmd}' at path {path_to} to spawn nodes.");
1360                        true
1361                    } else {
1362                        false
1363                    }
1364                })
1365            };
1366
1367            if missing {
1368                errs.push(help_msg(cmd));
1369            }
1370        }
1371    }
1372
1373    if !errs.is_empty() {
1374        let msg = errs.join("\n");
1375        return Err(anyhow::anyhow!(format!("Invalid configuration: \n {msg}")));
1376    }
1377
1378    Ok(())
1379}
1380
1381fn help_msg(cmd: &str) -> String {
1382    match cmd {
1383        "parachain-template-node" | "solochain-template-node" | "minimal-template-node" => {
1384            format!("Missing binary {cmd}, compile by running: \n\tcargo build --package {cmd} --release")
1385        },
1386        "polkadot" => {
1387            format!("Missing binary {cmd}, compile by running (in the polkadot-sdk repo): \n\t cargo build --locked --release --features fast-runtime --bin {cmd} --bin polkadot-prepare-worker --bin polkadot-execute-worker")
1388        },
1389        "polkadot-parachain" => {
1390            format!("Missing binary {cmd}, compile by running (in the polkadot-sdk repo): \n\t cargo build --release --locked -p {cmd}-bin --bin {cmd}")
1391        },
1392        _ => {
1393            format!("Missing binary {cmd}, please compile it.")
1394        },
1395    }
1396}
1397
1398/// Allow to set the default concurrency through env var `ZOMBIE_SPAWN_CONCURRENCY`
1399fn spawn_concurrency_from_env() -> Option<usize> {
1400    if let Ok(concurrency) = env::var("ZOMBIE_SPAWN_CONCURRENCY") {
1401        concurrency.parse::<usize>().ok()
1402    } else {
1403        None
1404    }
1405}
1406
1407/// The spawn concurrency as set (env over settings, 20 when neither), before
1408/// any token limit: what custom processes spawn with, since their placeholders
1409/// resolve from nodes that are all up and ports that are all reserved before
1410/// any starts. At least 1: with 0 nothing would ever start.
1411fn configured_concurrency(spec: &NetworkSpec) -> usize {
1412    spawn_concurrency_from_env()
1413        .or(spec.global_settings.spawn_concurrency())
1414        .unwrap_or(20)
1415        .max(1)
1416}
1417
1418/// Whether a chain's spec has a `{{ZOMBIE:..}}` placeholder, which makes the
1419/// nodes spawn one at a time. Custom processes are left out: theirs resolve
1420/// from nodes that are all up by then.
1421fn chains_have_tokens(spec: &NetworkSpec) -> Result<bool, anyhow::Error> {
1422    let chains = (&spec.relaychain, &spec.jamchain, &spec.parachains);
1423    Ok(has_tokens(&serde_json::to_string(&chains)?))
1424}
1425
1426/// The concurrency nodes spawn with, and whether tokens limited it to 1.
1427fn calculate_concurrency(spec: &NetworkSpec) -> Result<(usize, bool), anyhow::Error> {
1428    let concurrency = configured_concurrency(spec);
1429    if concurrency > 1 && chains_have_tokens(spec)? {
1430        Ok((1, true))
1431    } else {
1432        Ok((concurrency, false))
1433    }
1434}
1435
1436/// Build deterministic dependency **levels** among the given nodes.
1437/// - Only dependencies **between nodes in `nodes`** are considered.
1438/// - Unknown/out-of-scope references are ignored.
1439/// - Self-dependencies are ignored.
1440fn dependency_levels_among<'a>(
1441    nodes: &'a [&'a NodeSpec],
1442) -> Result<Vec<Vec<&'a NodeSpec>>, OrchestratorError> {
1443    let by_name = nodes
1444        .iter()
1445        .map(|n| (n.name.as_str(), *n))
1446        .collect::<HashMap<_, _>>();
1447
1448    let mut graph = HashMap::with_capacity(nodes.len());
1449    let mut indegree = HashMap::with_capacity(nodes.len());
1450
1451    for node in nodes {
1452        graph.insert(node.name.as_str(), Vec::new());
1453        indegree.insert(node.name.as_str(), 0);
1454    }
1455
1456    // build dependency graph
1457    for &node in nodes {
1458        if let Ok(args_json) = serde_json::to_string(&node.args) {
1459            // collect dependencies
1460            let unique_deps = get_tokens_to_replace(&args_json)
1461                .into_iter()
1462                .filter(|dep| dep != &node.name)
1463                .filter_map(|dep| by_name.get(dep.as_str()))
1464                .map(|&dep_node| dep_node.name.as_str())
1465                .collect::<HashSet<_>>();
1466
1467            for dep_name in unique_deps {
1468                graph
1469                    .get_mut(dep_name)
1470                    .expect(&format!("{GRAPH_CONTAINS_DEP} {THIS_IS_A_BUG}"))
1471                    .push(node);
1472                *indegree
1473                    .get_mut(node.name.as_str())
1474                    .expect(&format!("{INDEGREE_CONTAINS_NAME} {THIS_IS_A_BUG}")) += 1;
1475            }
1476        }
1477    }
1478
1479    // find all nodes with no dependencies
1480    let mut queue = nodes
1481        .iter()
1482        .filter(|n| {
1483            *indegree
1484                .get(n.name.as_str())
1485                .expect(&format!("{INDEGREE_CONTAINS_NAME} {THIS_IS_A_BUG}"))
1486                == 0
1487        })
1488        .copied()
1489        .collect::<VecDeque<_>>();
1490
1491    let mut processed_count = 0;
1492    let mut levels = Vec::new();
1493
1494    // Kahn's algorithm
1495    while !queue.is_empty() {
1496        let level_size = queue.len();
1497        let mut current_level = Vec::with_capacity(level_size);
1498
1499        for _ in 0..level_size {
1500            let n = queue
1501                .pop_front()
1502                .expect(&format!("{QUEUE_NOT_EMPTY} {THIS_IS_A_BUG}"));
1503            current_level.push(n);
1504            processed_count += 1;
1505
1506            for &neighbour in graph
1507                .get(n.name.as_str())
1508                .expect(&format!("{GRAPH_CONTAINS_NAME} {THIS_IS_A_BUG}"))
1509            {
1510                let neighbour_indegree = indegree
1511                    .get_mut(neighbour.name.as_str())
1512                    .expect(&format!("{INDEGREE_CONTAINS_NAME} {THIS_IS_A_BUG}"));
1513                *neighbour_indegree -= 1;
1514
1515                if *neighbour_indegree == 0 {
1516                    queue.push_back(neighbour);
1517                }
1518            }
1519        }
1520
1521        current_level.sort_by_key(|n| &n.name);
1522        levels.push(current_level);
1523    }
1524
1525    // cycles detected, e.g A -> B -> A
1526    if processed_count != nodes.len() {
1527        return Err(OrchestratorError::InvalidConfig(
1528            "Tokens have cyclical dependencies".to_string(),
1529        ));
1530    }
1531
1532    Ok(levels)
1533}
1534
1535// TODO: get the fs from `DynNamespace` will make this not needed
1536// but the FileSystem trait isn't object-safe so we can't pass around
1537// as `dyn FileSystem`. We can refactor or using some `erase` techniques
1538// to resolve this and remove this struct
1539// TODO (Loris): Probably we could have a .scoped(base_dir) method on the
1540// filesystem itself (the trait), so it will return this and we can move this
1541// directly to the support crate, it can be useful in the future
1542#[derive(Clone, Debug)]
1543pub struct ScopedFilesystem<'a, FS: FileSystem> {
1544    fs: &'a FS,
1545    base_dir: &'a str,
1546}
1547
1548impl<'a, FS: FileSystem> ScopedFilesystem<'a, FS> {
1549    pub fn new(fs: &'a FS, base_dir: &'a str) -> Self {
1550        Self { fs, base_dir }
1551    }
1552
1553    async fn copy_files(&self, files: Vec<&TransferedFile>) -> Result<(), FileSystemError> {
1554        for file in files {
1555            let full_remote_path = PathBuf::from(format!(
1556                "{}/{}",
1557                self.base_dir,
1558                file.remote_path.to_string_lossy()
1559            ));
1560            trace!("coping file: {file}");
1561            self.fs
1562                .copy(file.local_path.as_path(), full_remote_path)
1563                .await?;
1564        }
1565        Ok(())
1566    }
1567
1568    async fn read(&self, file: impl AsRef<Path>) -> Result<Vec<u8>, FileSystemError> {
1569        let file = file.as_ref();
1570
1571        let full_path = if file.is_absolute() {
1572            file.to_owned()
1573        } else {
1574            PathBuf::from(format!("{}/{}", self.base_dir, file.to_string_lossy()))
1575        };
1576        let content = self.fs.read(full_path).await?;
1577        Ok(content)
1578    }
1579
1580    async fn read_to_string(&self, file: impl AsRef<Path>) -> Result<String, FileSystemError> {
1581        let file = file.as_ref();
1582
1583        let full_path = if file.is_absolute() {
1584            file.to_owned()
1585        } else {
1586            PathBuf::from(format!("{}/{}", self.base_dir, file.to_string_lossy()))
1587        };
1588        let content = self.fs.read_to_string(full_path).await?;
1589        Ok(content)
1590    }
1591
1592    async fn create_dir(&self, path: impl AsRef<Path>) -> Result<(), FileSystemError> {
1593        let path = PathBuf::from(format!(
1594            "{}/{}",
1595            self.base_dir,
1596            path.as_ref().to_string_lossy()
1597        ));
1598        self.fs.create_dir(path).await
1599    }
1600
1601    async fn create_dir_all(&self, path: impl AsRef<Path>) -> Result<(), FileSystemError> {
1602        let path = PathBuf::from(format!(
1603            "{}/{}",
1604            self.base_dir,
1605            path.as_ref().to_string_lossy()
1606        ));
1607        self.fs.create_dir_all(path).await
1608    }
1609
1610    async fn write(
1611        &self,
1612        path: impl AsRef<Path>,
1613        contents: impl AsRef<[u8]> + Send,
1614    ) -> Result<(), FileSystemError> {
1615        let path = path.as_ref();
1616
1617        let full_path = if path.is_absolute() {
1618            path.to_owned()
1619        } else {
1620            PathBuf::from(format!("{}/{}", self.base_dir, path.to_string_lossy()))
1621        };
1622
1623        self.fs.write(full_path, contents).await
1624    }
1625
1626    /// Get the full_path in the scoped FS
1627    fn full_path(&self, path: impl AsRef<Path>) -> PathBuf {
1628        let path = path.as_ref();
1629
1630        let full_path = if path.is_absolute() {
1631            path.to_owned()
1632        } else {
1633            PathBuf::from(format!("{}/{}", self.base_dir, path.to_string_lossy()))
1634        };
1635
1636        full_path
1637    }
1638
1639    /// Get the base_dir in the scoped FS
1640    fn base_dir(&self) -> &str {
1641        self.base_dir
1642    }
1643}
1644
1645#[derive(Clone, Debug)]
1646pub enum ZombieRole {
1647    Temp,
1648    Node,
1649    Bootnode,
1650    Collator,
1651    CumulusCollator,
1652}
1653
1654impl ZombieRole {
1655    pub(crate) fn node_role(&self) -> NodeRole {
1656        match self {
1657            ZombieRole::Node => NodeRole::Node,
1658            ZombieRole::Bootnode => NodeRole::Bootnode,
1659            ZombieRole::Collator => NodeRole::Collator,
1660            ZombieRole::CumulusCollator => NodeRole::CumulusCollator,
1661            ZombieRole::Temp => NodeRole::Temp,
1662        }
1663    }
1664}
1665
1666// re-exports
1667pub use network::{AddCollatorOptions, AddNodeOptions};
1668pub use network_helper::metrics;
1669pub use sc_chain_spec;
1670
1671#[cfg(test)]
1672mod tests {
1673    use configuration::{GlobalSettingsBuilder, NetworkConfigBuilder};
1674    use lazy_static::lazy_static;
1675    use tokio::sync::Mutex;
1676
1677    use super::*;
1678
1679    const ENV_KEY: &str = "ZOMBIE_SPAWN_CONCURRENCY";
1680    // mutex for test that use env
1681    lazy_static! {
1682        static ref ENV_MUTEX: Mutex<()> = Mutex::new(());
1683    }
1684
1685    fn set_env(concurrency: Option<u32>) {
1686        if let Some(value) = concurrency {
1687            env::set_var(ENV_KEY, value.to_string());
1688        } else {
1689            env::remove_var(ENV_KEY);
1690        }
1691    }
1692
1693    fn generate(
1694        with_image: bool,
1695        with_cmd: Option<&'static str>,
1696    ) -> Result<NetworkConfig, Vec<anyhow::Error>> {
1697        NetworkConfigBuilder::new()
1698            .with_relaychain(|r| {
1699                let mut relay = r
1700                    .with_chain("rococo-local")
1701                    .with_default_command(with_cmd.unwrap_or("polkadot"));
1702                if with_image {
1703                    relay = relay.with_default_image("docker.io/parity/polkadot")
1704                }
1705
1706                relay
1707                    .with_validator(|node| node.with_name("alice"))
1708                    .with_validator(|node| node.with_name("bob"))
1709            })
1710            .with_parachain(|p| {
1711                p.with_id(2000).cumulus_based(true).with_collator(|n| {
1712                    let node = n
1713                        .with_name("collator")
1714                        .with_command(with_cmd.unwrap_or("polkadot-parachain"));
1715                    if with_image {
1716                        node.with_image("docker.io/paritypr/test-parachain")
1717                    } else {
1718                        node
1719                    }
1720                })
1721            })
1722            .build()
1723    }
1724
1725    fn get_node_with_dependencies(name: &str, dependencies: Option<Vec<&NodeSpec>>) -> NodeSpec {
1726        let mut spec = NodeSpec {
1727            name: name.to_string(),
1728            ..Default::default()
1729        };
1730        if let Some(dependencies) = dependencies {
1731            for node in dependencies {
1732                spec.args.push(
1733                    format!("{{{{ZOMBIE:{}:someField}}}}", node.name)
1734                        .as_str()
1735                        .into(),
1736                );
1737            }
1738        }
1739        spec
1740    }
1741
1742    fn verify_levels(actual_levels: Vec<Vec<&NodeSpec>>, expected_levels: Vec<Vec<&str>>) {
1743        actual_levels
1744            .iter()
1745            .zip(expected_levels)
1746            .for_each(|(actual_level, expected_level)| {
1747                assert_eq!(actual_level.len(), expected_level.len());
1748                actual_level
1749                    .iter()
1750                    .zip(expected_level.iter())
1751                    .for_each(|(node, expected_name)| assert_eq!(node.name, *expected_name));
1752            });
1753    }
1754
1755    #[tokio::test]
1756    async fn valid_config_with_image() {
1757        let network_config = generate(true, None).unwrap();
1758        let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1759        let caps = ProviderCapabilities {
1760            requires_image: true,
1761            has_resources: false,
1762            prefix_with_full_path: false,
1763            use_default_ports_in_cmd: false,
1764            has_secrets: false,
1765            fileserver_fetches_db_snapshot_url: false,
1766        };
1767
1768        let valid = validate_spec_with_provider_capabilities(&spec, &caps);
1769        assert!(valid.is_ok())
1770    }
1771
1772    #[tokio::test]
1773    async fn invalid_config_without_image() {
1774        let network_config = generate(false, None).unwrap();
1775        let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1776        let caps = ProviderCapabilities {
1777            requires_image: true,
1778            has_resources: false,
1779            prefix_with_full_path: false,
1780            use_default_ports_in_cmd: false,
1781            has_secrets: false,
1782            fileserver_fetches_db_snapshot_url: false,
1783        };
1784
1785        let valid = validate_spec_with_provider_capabilities(&spec, &caps);
1786        assert!(valid.is_err())
1787    }
1788
1789    #[tokio::test]
1790    async fn invalid_config_missing_cmd() {
1791        let network_config = generate(false, Some("other")).unwrap();
1792        let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1793        let caps = ProviderCapabilities {
1794            requires_image: false,
1795            has_resources: false,
1796            prefix_with_full_path: false,
1797            use_default_ports_in_cmd: false,
1798            has_secrets: false,
1799            fileserver_fetches_db_snapshot_url: false,
1800        };
1801
1802        let valid = validate_spec_with_provider_capabilities(&spec, &caps);
1803        assert!(valid.is_err())
1804    }
1805
1806    #[tokio::test]
1807    async fn valid_config_present_cmd() {
1808        let network_config = generate(false, Some("cargo")).unwrap();
1809        let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1810        let caps = ProviderCapabilities {
1811            requires_image: false,
1812            has_resources: false,
1813            prefix_with_full_path: false,
1814            use_default_ports_in_cmd: false,
1815            has_secrets: false,
1816            fileserver_fetches_db_snapshot_url: false,
1817        };
1818
1819        let valid = validate_spec_with_provider_capabilities(&spec, &caps);
1820        println!("{valid:?}");
1821        assert!(valid.is_ok())
1822    }
1823
1824    /// A provider that needs images and has no Secrets, as docker does.
1825    fn image_caps() -> ProviderCapabilities {
1826        ProviderCapabilities {
1827            requires_image: true,
1828            has_resources: false,
1829            prefix_with_full_path: false,
1830            use_default_ports_in_cmd: true,
1831            has_secrets: false,
1832            fileserver_fetches_db_snapshot_url: false,
1833        }
1834    }
1835
1836    /// A spec with one custom process `api` injecting `local` at `remote`.
1837    async fn spec_with_file(local: &str, remote: &str) -> NetworkSpec {
1838        let network_config = generate(true, None).unwrap();
1839        let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1840        spec.custom_processes.push(
1841            configuration::CustomProcessBuilder::new()
1842                .with_name("api")
1843                .with_command("api")
1844                .with_injected_file(configuration::TransferedFile::new(local, remote))
1845                .build()
1846                .unwrap(),
1847        );
1848        spec
1849    }
1850
1851    const AN_EXISTING_FILE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml");
1852
1853    #[tokio::test]
1854    async fn secret_env_needs_a_provider_with_secrets() {
1855        let network_config = generate(true, None).unwrap();
1856        let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1857        spec.custom_processes.push(
1858            configuration::CustomProcessBuilder::new()
1859                .with_name("api")
1860                .with_command("api")
1861                .with_env(vec![configuration::ProcessEnvVar::from_secret(
1862                    "TURN_SECRET",
1863                    "turn",
1864                    "secret",
1865                )])
1866                .build()
1867                .unwrap(),
1868        );
1869
1870        let err = validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap_err();
1871        assert!(
1872            err.to_string()
1873                .contains("custom_processes['api'].env: 'TURN_SECRET' is read from a Secret"),
1874            "{err}"
1875        );
1876        let with = ProviderCapabilities {
1877            has_secrets: true,
1878            ..image_caps()
1879        };
1880        validate_spec_with_provider_capabilities(&spec, &with).unwrap();
1881    }
1882
1883    #[tokio::test]
1884    async fn injected_files_must_exist_before_anything_spawns() {
1885        let spec = spec_with_file("./nowhere/init.sql", "/init.sql").await;
1886        let err = validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap_err();
1887        assert!(
1888            err.to_string().contains(
1889                "custom_processes['api'].injected_files: no file at './nowhere/init.sql' (relative to"
1890            ),
1891            "{err}"
1892        );
1893
1894        let spec = spec_with_file(AN_EXISTING_FILE, "/init.sql").await;
1895        validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap();
1896    }
1897
1898    #[tokio::test]
1899    async fn injected_files_may_not_replace_a_mounted_directory() {
1900        let spec = spec_with_file(AN_EXISTING_FILE, "/data").await;
1901        let err = validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap_err();
1902        assert!(
1903            err.to_string().ends_with(
1904                "custom_processes['api'].injected_files: '/data' is a directory zombienet mounts"
1905            ),
1906            "{err}"
1907        );
1908
1909        let spec = spec_with_file(AN_EXISTING_FILE, "/data/init.sql").await;
1910        validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap();
1911
1912        let spec = spec_with_file(AN_EXISTING_FILE, "/scripts/run.sh").await;
1913        let err = validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap_err();
1914        assert!(
1915            err.to_string().ends_with(
1916                "custom_processes['api'].injected_files: '/scripts/run.sh' is inside /scripts, which the containers of a network share"
1917            ),
1918            "{err}"
1919        );
1920    }
1921
1922    #[tokio::test]
1923    async fn default_spawn_concurrency() {
1924        let _g = ENV_MUTEX.lock().await;
1925        set_env(None);
1926        let network_config = generate(false, Some("cargo")).unwrap();
1927        let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1928        let (concurrency, _) = calculate_concurrency(&spec).unwrap();
1929        assert_eq!(concurrency, 20);
1930    }
1931
1932    #[tokio::test]
1933    async fn set_spawn_concurrency() {
1934        let _g = ENV_MUTEX.lock().await;
1935        set_env(None);
1936
1937        let network_config = generate(false, Some("cargo")).unwrap();
1938        let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1939
1940        let global_settings = GlobalSettingsBuilder::new()
1941            .with_spawn_concurrency(4)
1942            .build()
1943            .unwrap();
1944
1945        spec.set_global_settings(global_settings);
1946        let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
1947        assert_eq!(concurrency, 4);
1948        assert!(!limited);
1949    }
1950
1951    #[tokio::test]
1952    async fn set_spawn_concurrency_but_limited() {
1953        let _g = ENV_MUTEX.lock().await;
1954        set_env(None);
1955
1956        let network_config = generate(false, Some("cargo")).unwrap();
1957        let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1958
1959        let global_settings = GlobalSettingsBuilder::new()
1960            .with_spawn_concurrency(4)
1961            .build()
1962            .unwrap();
1963
1964        spec.set_global_settings(global_settings);
1965        let node = spec.relaychain_mut().nodes.first_mut().unwrap();
1966        node.args
1967            .push("--bootnodes {{ZOMBIE:bob:multiAddress')}}".into());
1968        let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
1969        assert_eq!(concurrency, 1);
1970        assert!(limited);
1971    }
1972
1973    #[tokio::test]
1974    async fn custom_process_placeholders_do_not_limit_spawn_concurrency() {
1975        let _g = ENV_MUTEX.lock().await;
1976        set_env(None);
1977
1978        let network_config = generate(false, Some("cargo")).unwrap();
1979        let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1980
1981        let global_settings = GlobalSettingsBuilder::new()
1982            .with_spawn_concurrency(4)
1983            .build()
1984            .unwrap();
1985
1986        spec.set_global_settings(global_settings);
1987        spec.custom_processes.push(
1988            configuration::CustomProcessBuilder::new()
1989                .with_name("api")
1990                .with_command("api")
1991                .with_args(vec!["{{ZOMBIE:alice:ws_uri}}".into()])
1992                .build()
1993                .unwrap(),
1994        );
1995        let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
1996        assert_eq!(concurrency, 4);
1997        assert!(!limited);
1998    }
1999
2000    #[tokio::test]
2001    async fn set_spawn_concurrency_from_env() {
2002        let _g = ENV_MUTEX.lock().await;
2003        set_env(Some(10));
2004
2005        let network_config = generate(false, Some("cargo")).unwrap();
2006        let spec = NetworkSpec::from_config(&network_config).await.unwrap();
2007        let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
2008        assert_eq!(concurrency, 10);
2009        assert!(!limited);
2010    }
2011
2012    #[tokio::test]
2013    async fn set_spawn_concurrency_from_env_but_limited() {
2014        let _g = ENV_MUTEX.lock().await;
2015        set_env(Some(12));
2016
2017        let network_config = generate(false, Some("cargo")).unwrap();
2018        let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
2019        let node = spec.relaychain_mut().nodes.first_mut().unwrap();
2020        node.args
2021            .push("--bootnodes {{ZOMBIE:bob:multiAddress')}}".into());
2022        let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
2023        assert_eq!(concurrency, 1);
2024        assert!(limited);
2025    }
2026
2027    #[test]
2028    fn dependency_levels_among_should_work() {
2029        // no nodes
2030        assert!(dependency_levels_among(&[]).unwrap().is_empty());
2031
2032        // one node
2033        let alice = get_node_with_dependencies("alice", None);
2034        let nodes = [&alice];
2035
2036        let levels = dependency_levels_among(&nodes).unwrap();
2037        let expected = vec![vec!["alice"]];
2038
2039        verify_levels(levels, expected);
2040
2041        // two independent nodes
2042        let alice = get_node_with_dependencies("alice", None);
2043        let bob = get_node_with_dependencies("bob", None);
2044        let nodes = [&alice, &bob];
2045
2046        let levels = dependency_levels_among(&nodes).unwrap();
2047        let expected = vec![vec!["alice", "bob"]];
2048
2049        verify_levels(levels, expected);
2050
2051        // alice -> bob -> charlie
2052        let alice = get_node_with_dependencies("alice", None);
2053        let bob = get_node_with_dependencies("bob", Some(vec![&alice]));
2054        let charlie = get_node_with_dependencies("charlie", Some(vec![&bob]));
2055        let nodes = [&alice, &bob, &charlie];
2056
2057        let levels = dependency_levels_among(&nodes).unwrap();
2058        let expected = vec![vec!["alice"], vec!["bob"], vec!["charlie"]];
2059
2060        verify_levels(levels, expected);
2061
2062        //         ┌─> bob
2063        // alice ──|
2064        //         └─> charlie
2065        let alice = get_node_with_dependencies("alice", None);
2066        let bob = get_node_with_dependencies("bob", Some(vec![&alice]));
2067        let charlie = get_node_with_dependencies("charlie", Some(vec![&alice]));
2068        let nodes = [&alice, &bob, &charlie];
2069
2070        let levels = dependency_levels_among(&nodes).unwrap();
2071        let expected = vec![vec!["alice"], vec!["bob", "charlie"]];
2072
2073        verify_levels(levels, expected);
2074
2075        //         ┌─>   bob  ──┐
2076        // alice ──|            ├─> dave
2077        //         └─> charlie  ┘
2078        let alice = get_node_with_dependencies("alice", None);
2079        let bob = get_node_with_dependencies("bob", Some(vec![&alice]));
2080        let charlie = get_node_with_dependencies("charlie", Some(vec![&alice]));
2081        let dave = get_node_with_dependencies("dave", Some(vec![&charlie, &bob]));
2082        let nodes = [&alice, &bob, &charlie, &dave];
2083
2084        let levels = dependency_levels_among(&nodes).unwrap();
2085        let expected = vec![vec!["alice"], vec!["bob", "charlie"], vec!["dave"]];
2086
2087        verify_levels(levels, expected);
2088    }
2089
2090    #[test]
2091    fn dependency_levels_among_should_detect_cycles() {
2092        let mut alice = get_node_with_dependencies("alice", None);
2093        let bob = get_node_with_dependencies("bob", Some(vec![&alice]));
2094        alice.args.push("{{ZOMBIE:bob:someField}}".into());
2095
2096        assert!(dependency_levels_among(&[&alice, &bob]).is_err())
2097    }
2098}