1#![allow(dead_code, clippy::expect_fun_call)]
3
4pub mod errors;
5pub mod generators;
6pub mod network;
7pub mod network_helper;
8pub mod observability;
9pub mod tx_helper;
10
11mod custom_processes;
12mod network_spec;
13pub mod shared;
14mod spawner;
15mod utils;
16
17use std::{
18 collections::{HashMap, HashSet, VecDeque},
19 env,
20 net::IpAddr,
21 path::{Path, PathBuf},
22 sync::Arc,
23 time::{Duration, SystemTime},
24};
25
26use anyhow::anyhow;
27use configuration::{types::JsonOverrides, NetworkConfig, RegistrationStrategy};
28use errors::OrchestratorError;
29use generators::{core_assignment, errors::GeneratorError, jam_chain_spec};
30use network::{
31 node::{NetworkNode, SpawnedNode},
32 parachain::Parachain,
33 relaychain::Relaychain,
34 Network,
35};
36pub use network_spec::NetworkSpec;
38use network_spec::{
39 jamchain::{JamChainSpecSource, JamchainSpec},
40 node::NodeSpec,
41 parachain::ParachainSpec,
42};
43use provider::{
44 constants::{HELPERS_DIR, NODE_MOUNTED_DIRS, NODE_SCRIPTS_DIR},
45 shared::helpers::running_in_ci,
46 types::{NodeRole, ProviderCapabilities, TransferedFile},
47 DynNamespace, DynProvider,
48};
49use serde_json::json;
50use support::{
51 constants::{
52 GRAPH_CONTAINS_DEP, GRAPH_CONTAINS_NAME, INDEGREE_CONTAINS_NAME, QUEUE_NOT_EMPTY,
53 THIS_IS_A_BUG,
54 },
55 fs::{FileSystem, FileSystemError},
56 replacer::{get_tokens_to_replace, has_tokens},
57};
58use tokio::time::timeout;
59use tracing::{debug, info, trace, warn};
60
61use crate::{
62 network::{
63 jamchain::{Jamchain, RawJamchain},
64 node::{
65 jam::RawJamNetworkNode, process::RawCustomProcessNode, CustomProcessNode,
66 CustomProcessState, JamNetworkNode, RawNetworkNode,
67 },
68 parachain::RawParachain,
69 relaychain::RawRelaychain,
70 },
71 shared::types::RegisterParachainOptions,
72 spawner::SpawnNodeCtx,
73 utils::write_zombie_json,
74};
75pub struct Orchestrator<T>
76where
77 T: FileSystem + Sync + Send,
78{
79 filesystem: T,
80 provider: DynProvider,
81}
82
83impl<T> Orchestrator<T>
84where
85 T: FileSystem + Sync + Send + Clone,
86{
87 pub fn new(filesystem: T, provider: DynProvider) -> Self {
88 Self {
89 filesystem,
90 provider,
91 }
92 }
93
94 pub async fn spawn(
95 &self,
96 network_config: NetworkConfig,
97 ) -> Result<Network<T>, OrchestratorError> {
98 let global_timeout = network_config.global_settings().network_spawn_timeout();
99 let network_spec = NetworkSpec::from_config(&network_config).await?;
100
101 let res = timeout(
102 Duration::from_secs(global_timeout.into()),
103 self.spawn_inner(network_spec),
104 )
105 .await
106 .map_err(|_| OrchestratorError::GlobalTimeOut(global_timeout));
107 res?
108 }
109
110 pub async fn spawn_from_spec(
111 &self,
112 network_spec: NetworkSpec,
113 ) -> Result<Network<T>, OrchestratorError> {
114 let global_timeout = network_spec.global_settings.network_spawn_timeout();
115 let res = timeout(
116 Duration::from_secs(global_timeout as u64),
117 self.spawn_inner(network_spec),
118 )
119 .await
120 .map_err(|_| OrchestratorError::GlobalTimeOut(global_timeout));
121 res?
122 }
123
124 pub async fn attach_to_live(
125 &self,
126 zombie_json_path: &Path,
127 ) -> Result<Network<T>, OrchestratorError> {
128 info!("attaching to live network...");
129 info!("reading zombie.json from {:?}", zombie_json_path);
130
131 let zombie_json_content = self.filesystem.read_to_string(zombie_json_path).await?;
132 let zombie_json: serde_json::Value = serde_json::from_str(&zombie_json_content)?;
133
134 self.attach_to_live_from_json(&zombie_json).await
135 }
136
137 pub async fn attach_to_live_from_json(
140 &self,
141 zombie_json: &serde_json::Value,
142 ) -> Result<Network<T>, OrchestratorError> {
143 info!("recreating namespace...");
144 let ns: DynNamespace = self
145 .provider
146 .create_namespace_from_json(zombie_json)
147 .await?;
148
149 info!("recreating relaychain...");
150 let (relay, initial_spec) =
151 recreate_relaychain_from_json(zombie_json, ns.clone(), self.provider.name()).await?;
152 let relay_nodes = relay.nodes.clone();
153
154 let mut network =
155 Network::new_with_relay(relay, ns.clone(), self.filesystem.clone(), initial_spec);
156
157 for node in relay_nodes {
158 if node.is_responsive().await {
159 node.set_is_running(true);
160 }
161 network.insert_node(node);
162 }
163
164 info!("recreating parachains...");
165 let parachains_map =
166 recreate_parachains_from_json(zombie_json, ns.clone(), self.provider.name()).await?;
167 let para_nodes = parachains_map
168 .values()
169 .flat_map(|paras| paras.iter().flat_map(|para| para.collators.clone()))
170 .collect::<Vec<Arc<NetworkNode>>>();
171
172 network.set_parachains(parachains_map);
173 for node in para_nodes {
174 if node.is_responsive().await {
175 node.set_is_running(true);
176 }
177 network.insert_node(node);
178 }
179
180 if let Some(jamchain) =
181 recreate_jamchain_from_json(zombie_json, ns.clone(), self.provider.name()).await?
182 {
183 info!("recreating jamchain...");
184 let jam_nodes = jamchain.nodes.clone();
185 network.set_jamchain(jamchain);
186 for node in jam_nodes {
187 if node.is_responsive().await {
188 node.core().set_is_running(true);
189 }
190 network.insert_node(node);
191 }
192 }
193
194 let processes =
195 recreate_custom_processes_from_json(zombie_json, ns.clone(), self.provider.name())
196 .await?;
197 if !processes.is_empty() {
198 info!("recreating custom processes...");
199 for process in &processes {
200 if process.is_responsive().await {
203 process.core().set_is_running(true);
204 }
205 }
206 network.set_custom_processes(processes);
207 }
208
209 Ok(network)
210 }
211
212 async fn spawn_inner(
213 &self,
214 mut network_spec: NetworkSpec,
215 ) -> Result<Network<T>, OrchestratorError> {
216 debug!(network_spec = ?network_spec,"Network spec to spawn");
218
219 validate_spec_with_provider_capabilities(&network_spec, self.provider.capabilities())
221 .map_err(|err| {
222 OrchestratorError::InvalidConfigForProvider(
223 self.provider.name().into(),
224 err.to_string(),
225 )
226 })?;
227
228 let ns = if let Some(base_dir) = network_spec.global_settings.base_dir() {
230 self.provider
231 .create_namespace_with_base_dir(base_dir)
232 .await?
233 } else {
234 self.provider.create_namespace().await?
235 };
236
237 let (spawn_concurrency, limited_by_tokens) = calculate_concurrency(&network_spec)?;
239
240 let start_time = SystemTime::now();
241 info!("🧰 ns: {}", ns.name());
242 info!("🧰 base_dir: {:?}", ns.base_dir());
243 info!("🕰 start time: {:?}", start_time);
244 info!("⚙️ spawn concurrency: {spawn_concurrency} (limited by tokens: {limited_by_tokens})");
245
246 network_spec
247 .populate_nodes_available_args(ns.clone())
248 .await?;
249
250 if network_spec.try_relaychain().is_none() {
253 let jam_spec =
254 network_spec
255 .jamchain
256 .clone()
257 .ok_or(OrchestratorError::InvalidConfig(
258 "A network needs either a relaychain or a jamchain.".to_string(),
259 ))?;
260
261 return self
262 .spawn_jam(&jam_spec, &mut network_spec, ns, start_time)
263 .await;
264 }
265
266 let all_nodes: Vec<&NodeSpec> = network_spec
269 .relaychain()
270 .nodes
271 .iter()
272 .chain(
273 network_spec
274 .parachains_iter()
275 .flat_map(|p| p.collators.iter()),
276 )
277 .collect();
278 let resolved_db_snapshots =
279 generators::resolve_db_snapshots(all_nodes, &ns, &self.filesystem).await?;
280
281 let base_dir = ns.base_dir().to_string_lossy();
282 let scoped_fs = ScopedFilesystem::new(&self.filesystem, &base_dir);
283
284 let mut relaychain = network_spec
288 .relaychain
289 .take()
290 .expect("checked to be `Some` above; qed");
291
292 relaychain.chain_spec.build(&ns, &scoped_fs).await?;
294
295 debug!("relaychain spec built!");
296 let relay_chain_id = relaychain.chain_spec.read_chain_id(&scoped_fs).await?;
298
299 let relay_chain_name = relaychain.chain.as_str().to_owned();
300 let base_dir_exists = network_spec.global_settings.base_dir().is_some();
301 network_spec
302 .build_parachain_artifacts(ns.clone(), &scoped_fs, &relay_chain_id, base_dir_exists)
303 .await?;
304
305 let (para_to_register_in_genesis, para_to_register_with_extrinsic): (
307 Vec<&ParachainSpec>,
308 Vec<&ParachainSpec>,
309 ) = network_spec
310 .parachains
311 .iter()
312 .filter(|para| para.registration_strategy != RegistrationStrategy::Manual)
313 .partition(|para| {
314 matches!(para.registration_strategy, RegistrationStrategy::InGenesis)
315 });
316
317 let mut para_artifacts = vec![];
318 for para in ¶_to_register_in_genesis {
319 let genesis_config = para.get_genesis_config()?;
320 para_artifacts.push(genesis_config)
321 }
322
323 relaychain
325 .chain_spec
326 .customize_relay(
327 &relaychain,
328 &network_spec.hrmp_channels,
329 para_artifacts,
330 &scoped_fs,
331 )
332 .await?;
333
334 if let Some(script_cmd) = relaychain.post_process_script.as_deref() {
336 relaychain
337 .chain_spec
338 .run_post_process_script(script_cmd, &scoped_fs)
339 .await?;
340 }
341
342 let num_cores = network_spec.parachains_iter().fold(0u32, |mut acc, para| {
344 if let Some(cores) = para.num_cores {
345 acc += cores;
346 } else if &RegistrationStrategy::InGenesis == para.registration_strategy() {
347 acc += 1;
349 }
350
351 acc
352 });
353
354 if num_cores > para_to_register_in_genesis.len() as u32 {
355 let num_cores_to_set = num_cores - para_to_register_in_genesis.len() as u32;
356 let overrides = json!({
358 "configuration": {
359 "config": {
360 "scheduler_params": {
361 "num_cores": num_cores_to_set,
362 "max_validators_per_core": 1
363 },
364 }
365 }
366 });
367
368 relaychain
369 .chain_spec
370 .apply_genesis_override(&scoped_fs, &overrides)
371 .await?;
372 }
373
374 relaychain
376 .chain_spec
377 .build_raw(&ns, &scoped_fs, None)
378 .await?;
379
380 if let Some(ref wasm_override) = relaychain.wasm_override {
382 relaychain
383 .chain_spec
384 .override_code(&scoped_fs, wasm_override)
385 .await?;
386 }
387
388 if let Some(ref raw_spec_override) = relaychain.raw_spec_override {
390 relaychain
391 .chain_spec
392 .override_raw_spec(&scoped_fs, raw_spec_override)
393 .await?;
394 }
395
396 if num_cores > para_to_register_in_genesis.len() as u32
398 || (relaychain.override_session_0 && !para_to_register_in_genesis.is_empty())
399 {
400 debug!("Raw overrides info: num_cores: {}, para_to_register_in_genesis_len: {:?}, override_session_0: {}", num_cores, para_to_register_in_genesis.len(), relaychain.override_session_0);
401 let mut core_index = 0u32;
402 let scheduler_key = core_assignment::get_parascheduler_storage_key();
405 let is_old = !relaychain
406 .chain_spec
407 .find_raw_key(&scoped_fs, &scheduler_key)
408 .await?;
409
410 let mut para_scheduler_value_parts: Vec<String> = vec![];
411 let mut raw_json_overrides = json!({});
412 for para in &network_spec.parachains {
414 let mut cores_for_para = 0_u32;
416 if let Some(cores) = para.num_cores {
417 if &RegistrationStrategy::InGenesis == para.registration_strategy() {
418 cores_for_para = cores;
419 }
420 } else {
421 if &RegistrationStrategy::InGenesis == para.registration_strategy()
424 && relaychain.override_session_0
425 {
426 cores_for_para += 1;
427 }
428 }
429
430 debug!(
431 "Assigning {cores_for_para} cores in raw spec for para {}. Using pallet {}",
432 para.id,
433 if is_old {
434 "CoretimeAssignmentProvider"
435 } else {
436 "ParaScheduler"
437 }
438 );
439 for _core in 0..cores_for_para {
440 if is_old {
441 let (core_assign_key, core_assign_value) =
442 core_assignment::generate_old(core_index, para.id);
443 raw_json_overrides[core_assign_key] = json!(core_assign_value);
444 } else {
445 let part = core_assignment::generate(core_index, para.id);
446 para_scheduler_value_parts.push(part);
447 }
448 core_index += 1;
449 }
450 }
451
452 if !is_old {
454 let count_prefix = format!("{:02x}", para_scheduler_value_parts.len() * 4);
455 let core_assign_value =
456 format!("{count_prefix}{}", para_scheduler_value_parts.join(""));
457 raw_json_overrides[scheduler_key] = json!(core_assign_value);
458 }
459
460 if relaychain.override_session_0 {
462 trace!("Overriding pallet ParaSessionInfo.session (0) to allow paras to produce blocks at first session.");
463 let raw_spec = relaychain.chain_spec.read_raw_spec(&scoped_fs).await?;
464 let overrides = generators::generate_session_0_overrides(&raw_spec, num_cores)?;
465
466 for (k, v) in overrides.as_object().ok_or(anyhow!(
467 "'generate_session_0_overrides' should be a valid json Object."
468 ))? {
469 raw_json_overrides[k] = v.clone();
470 }
471 }
472
473 debug!("Raw overrides keys: {:?}", raw_json_overrides);
474
475 relaychain
476 .chain_spec
477 .override_raw_spec(
478 &scoped_fs,
479 &JsonOverrides::Json(json!({
480 "genesis": {
481 "raw": {
482 "top": raw_json_overrides
483 }
484 }
485 })),
486 )
487 .await?;
488 }
489
490 network_spec.relaychain = Some(relaychain);
492
493 let (bootnodes, relaynodes) =
494 split_nodes_by_bootnodes(&network_spec.relaychain().nodes, false);
495
496 let mut ctx = SpawnNodeCtx {
498 chain_id: &relay_chain_id,
499 parachain_id: None,
500 chain: relay_chain_name.as_str(),
501 role: ZombieRole::Node,
502 ns: &ns,
503 scoped_fs: &scoped_fs,
504 parachain: None,
505 bootnodes_addr: &vec![],
506 wait_ready: false,
507 nodes_by_name: json!({}),
508 global_settings: &network_spec.global_settings,
509 resolved_db_snapshots: &resolved_db_snapshots,
510 };
511
512 let global_files_to_inject = vec![TransferedFile::new(
513 PathBuf::from(format!(
514 "{}/{relay_chain_name}.json",
515 ns.base_dir().to_string_lossy()
516 )),
517 PathBuf::from(format!("/cfg/{relay_chain_name}.json")),
518 )];
519
520 let r = Relaychain::new(
521 relay_chain_name.to_string(),
522 relay_chain_id.clone(),
523 PathBuf::from(network_spec.relaychain().chain_spec.raw_path().ok_or(
524 OrchestratorError::InvariantError("chain-spec raw path should be set now"),
525 )?),
526 );
527 let mut network =
528 Network::new_with_relay(r, ns.clone(), self.filesystem.clone(), network_spec.clone());
529
530 let mut node_ws_url: String = "".to_string();
532
533 let mut bootnodes_addr: Vec<String> = vec![];
535
536 for level in dependency_levels_among(&bootnodes)? {
537 let mut running_nodes_per_level = vec![];
538 for chunk in level.chunks(spawn_concurrency) {
539 let spawning_tasks = chunk
540 .iter()
541 .map(|node| spawner::spawn_node(node, global_files_to_inject.clone(), &ctx));
542
543 for node in futures::future::try_join_all(spawning_tasks).await? {
544 let bootnode_multiaddr = node.multiaddr();
545
546 bootnodes_addr.push(bootnode_multiaddr.to_string());
547
548 if node_ws_url.is_empty() {
550 node_ws_url.clone_from(&node.ws_uri)
551 }
552
553 running_nodes_per_level.push(node);
554 }
555 }
556 info!(
557 "🕰 waiting for level: {:?} to be up...",
558 level.iter().map(|n| n.name.clone()).collect::<Vec<_>>()
559 );
560
561 let waiting_tasks = running_nodes_per_level.iter().map(|node| {
563 node.wait_until_is_up(network_spec.global_settings.node_spawn_timeout())
564 });
565
566 let _ = futures::future::try_join_all(waiting_tasks).await?;
567
568 for node in running_nodes_per_level {
569 ctx.nodes_by_name[node.name().to_owned()] = serde_json::to_value(&node)?;
571 network.add_running_node(node, None).await;
572 }
573 }
574
575 network_spec
577 .relaychain()
578 .chain_spec
579 .add_bootnodes(&scoped_fs, &bootnodes_addr)
580 .await?;
581
582 ctx.bootnodes_addr = &bootnodes_addr;
583
584 upload_raw_spec(&ns, &network_spec.relaychain().chain_spec).await?;
586
587 for level in dependency_levels_among(&relaynodes)? {
588 let mut running_nodes_per_level = vec![];
589 for chunk in level.chunks(spawn_concurrency) {
590 let spawning_tasks = chunk
591 .iter()
592 .map(|node| spawner::spawn_node(node, global_files_to_inject.clone(), &ctx));
593
594 for node in futures::future::try_join_all(spawning_tasks).await? {
595 running_nodes_per_level.push(node);
596 }
597 }
598 info!(
599 "🕰 waiting for level: {:?} to be up...",
600 level.iter().map(|n| n.name.clone()).collect::<Vec<_>>()
601 );
602
603 let waiting_tasks = running_nodes_per_level.iter().map(|node| {
605 node.wait_until_is_up(network_spec.global_settings.network_spawn_timeout())
606 });
607
608 let _ = futures::future::try_join_all(waiting_tasks).await?;
609
610 for node in running_nodes_per_level {
611 ctx.nodes_by_name[node.name().to_owned()] = serde_json::to_value(&node)?;
612 network.add_running_node(node, None).await;
613 }
614 }
615
616 self.spawn_parachains(
618 &network_spec.parachains,
619 &ctx,
620 &global_files_to_inject,
621 spawn_concurrency,
622 &mut network,
623 )
624 .await?;
625
626 for para in para_to_register_with_extrinsic {
628 let register_para_options: RegisterParachainOptions = RegisterParachainOptions {
629 id: para.id,
630 wasm_path: para
632 .genesis_wasm
633 .artifact_path()
634 .ok_or(OrchestratorError::InvariantError(
635 "artifact path for wasm must be set at this point",
636 ))?
637 .to_path_buf(),
638 state_path: para
639 .genesis_state
640 .artifact_path()
641 .ok_or(OrchestratorError::InvariantError(
642 "artifact path for state must be set at this point",
643 ))?
644 .to_path_buf(),
645 node_ws_url: node_ws_url.clone(),
646 onboard_as_para: para.onboard_as_parachain,
647 seed: None, finalization: false,
649 };
650
651 Parachain::register(register_para_options, &scoped_fs).await?;
652 }
653
654 custom_processes::spawn_custom_processes(
655 &mut network,
656 &network_spec,
657 ns.clone(),
658 configured_concurrency(&network_spec),
659 )
660 .await?;
661
662 if network_spec.global_settings.observability().enabled() {
665 match network
666 .start_observability(network_spec.global_settings.observability())
667 .await
668 {
669 Ok(obs) => {
670 info!("📊 Prometheus URL: {}", obs.prometheus_url);
671 info!("📊 Grafana URL: {}", obs.grafana_url);
672 },
673 Err(e) => {
674 warn!("⚠️ Failed to spawn observability stack: {e}");
675 },
676 }
677 }
678
679 network.set_start_time_ts(start_time);
680
681 write_zombie_json(serde_json::to_value(&network)?, scoped_fs, ns.name()).await?;
682
683 if network_spec.global_settings.tear_down_on_failure() {
684 network.spawn_watching_task();
685 }
686
687 generators::cleanup_db_snapshot_cache(&resolved_db_snapshots).await;
688
689 Ok(network)
690 }
691
692 async fn spawn_parachains<'a>(
697 &self,
698 parachains: &[ParachainSpec],
699 ctx: &SpawnNodeCtx<'a, T>,
700 files_to_inject: &[TransferedFile],
701 spawn_concurrency: usize,
702 network: &mut Network<T>,
703 ) -> Result<(), OrchestratorError> {
704 let scoped_fs = ctx.scoped_fs;
705
706 for para in parachains {
707 let parachain = Parachain::from_spec(para, files_to_inject, scoped_fs).await?;
709 let parachain_id = parachain.chain_id.clone();
710
711 let (bootnodes, collators) =
712 split_nodes_by_bootnodes(¶.collators, para.no_default_bootnodes);
713
714 let mut ctx_para = SpawnNodeCtx {
716 parachain: Some(para),
717 parachain_id: parachain_id.as_deref(),
718 role: if para.is_cumulus_based {
719 ZombieRole::CumulusCollator
720 } else {
721 ZombieRole::Collator
722 },
723 bootnodes_addr: &vec![],
724 ..ctx.clone()
725 };
726
727 let mut bootnodes_addr: Vec<String> = vec![];
729 let mut running_nodes: Vec<NetworkNode> = vec![];
730
731 for level in dependency_levels_among(&bootnodes)? {
732 for node in self
733 .spawn_parachain_level(&level, ¶chain, &ctx_para, spawn_concurrency)
734 .await?
735 {
736 bootnodes_addr.push(node.multiaddr().to_string());
737 ctx_para.nodes_by_name[node.name().to_owned()] = serde_json::to_value(&node)?;
738 running_nodes.push(node);
739 }
740 }
741
742 if let Some(para_chain_spec) = para.chain_spec.as_ref() {
743 para_chain_spec
744 .add_bootnodes(scoped_fs, &bootnodes_addr)
745 .await?;
746
747 upload_raw_spec(ctx.ns, para_chain_spec).await?;
749 }
750
751 ctx_para.bootnodes_addr = &bootnodes_addr;
752
753 for level in dependency_levels_among(&collators)? {
755 for node in self
756 .spawn_parachain_level(&level, ¶chain, &ctx_para, spawn_concurrency)
757 .await?
758 {
759 ctx_para.nodes_by_name[node.name().to_owned()] = serde_json::to_value(&node)?;
760 running_nodes.push(node);
761 }
762 }
763
764 let running_para_id = parachain.para_id;
765 network.add_para(parachain);
766 for node in running_nodes {
767 network.add_running_node(node, Some(running_para_id)).await;
768 }
769 }
770
771 Ok(())
772 }
773
774 async fn spawn_parachain_level<'a>(
776 &self,
777 level: &[&NodeSpec],
778 parachain: &Parachain,
779 ctx: &SpawnNodeCtx<'a, T>,
780 spawn_concurrency: usize,
781 ) -> Result<Vec<NetworkNode>, OrchestratorError> {
782 let mut running_nodes = vec![];
783 for chunk in level.chunks(spawn_concurrency) {
784 let spawning_tasks = chunk
785 .iter()
786 .map(|node| spawner::spawn_node(node, parachain.files_to_inject.clone(), ctx));
787
788 running_nodes.extend(futures::future::try_join_all(spawning_tasks).await?);
789 }
790
791 info!(
792 "🕰 waiting for level: {:?} to be up...",
793 level.iter().map(|n| n.name.clone()).collect::<Vec<_>>()
794 );
795
796 let waiting_tasks = running_nodes
798 .iter()
799 .map(|node| node.wait_until_is_up(ctx.global_settings.network_spawn_timeout()));
800
801 let _ = futures::future::try_join_all(waiting_tasks).await?;
802
803 Ok(running_nodes)
804 }
805
806 async fn spawn_jam(
808 &self,
809 jam_spec: &JamchainSpec,
810 network_spec: &mut NetworkSpec,
811 ns: DynNamespace,
812 start_time: SystemTime,
813 ) -> Result<Network<T>, OrchestratorError> {
814 let base_dir = ns.base_dir().to_string_lossy().to_string();
815 let scoped_fs = ScopedFilesystem::new(&self.filesystem, &base_dir);
816
817 let jam_spec_full_path = format!("{}/{}", base_dir, "jam_spec.json");
818 match &jam_spec.chain_spec_source {
819 JamChainSpecSource::Path(location) => {
820 info!("📜 using JAM chain spec {location}");
821 let spec = location.get_asset().await.map_err(|e| {
822 GeneratorError::ChainSpecGeneration(format!("JAM chain spec {location}: {e}"))
823 })?;
824 scoped_fs.write("jam_spec.json", spec).await?;
825 },
826 JamChainSpecSource::JamBuilder(genesis) => {
827 info!("🛠️ building the JAM chain spec in process");
828 let built = jam_chain_spec::build(jam_spec, genesis).await?;
829 scoped_fs.write("jam_spec.json", built.spec).await?;
830 scoped_fs
831 .write(
832 "jam_genesis.json",
833 serde_json::to_string_pretty(&built.summary)?,
834 )
835 .await?;
836 },
837 }
838
839 let base_dir_exists = network_spec.global_settings.base_dir().is_some();
842 network_spec
843 .build_parachain_artifacts(
844 ns.clone(),
845 &scoped_fs,
846 jam_spec.id.as_str(),
847 base_dir_exists,
848 )
849 .await?;
850
851 let collators: Vec<&NodeSpec> = network_spec
854 .parachains_iter()
855 .flat_map(|para| para.collators.iter())
856 .collect();
857 let resolved_db_snapshots =
858 generators::resolve_db_snapshots(collators, &ns, &self.filesystem).await?;
859
860 let (spawn_concurrency, _) = calculate_concurrency(network_spec)?;
861 let global_settings = &network_spec.global_settings;
862
863 let jam_ctx = SpawnNodeCtx {
865 chain_id: jam_spec.id.as_str(),
866 parachain_id: None,
867 chain: jam_spec.id.as_str(),
868 role: ZombieRole::Node,
869 ns: &ns,
870 scoped_fs: &scoped_fs,
871 parachain: None,
872 bootnodes_addr: &vec![],
873 wait_ready: false,
874 nodes_by_name: json!({}),
875 global_settings,
876 resolved_db_snapshots: &resolved_db_snapshots,
877 };
878
879 let jam_global_files_to_inject = vec![TransferedFile::new(
880 PathBuf::from(format!("{base_dir}/jam_spec.json")),
881 PathBuf::from("/cfg/jam_spec.json"),
882 )];
883
884 let mut network =
885 Network::new_without_relay(ns.clone(), self.filesystem.clone(), network_spec.clone());
886
887 network.set_jamchain(Jamchain::new(
888 jam_spec.id.as_str(),
889 PathBuf::from(&jam_spec_full_path),
890 ));
891
892 let mut bootnodes_addr = vec![];
893 let mut nodes_by_name = json!({});
895 for jam_node in &jam_spec.nodes {
896 let jam_ctx = SpawnNodeCtx {
897 bootnodes_addr: &bootnodes_addr,
898 nodes_by_name: nodes_by_name.clone(),
899 ..jam_ctx.clone()
900 };
901
902 let running_node =
903 spawner::spawn_jam_node(jam_node, jam_global_files_to_inject.clone(), &jam_ctx)
904 .await?;
905
906 running_node
907 .wait_until_is_up(global_settings.node_spawn_timeout() as u64)
908 .await
909 .map_err(|e| OrchestratorError::InvalidConfig(e.to_string()))?;
910
911 bootnodes_addr.push(running_node.peer_addr().to_string());
912 nodes_by_name[running_node.name().to_owned()] = serde_json::to_value(&running_node)?;
913 network.add_running_jam_node(running_node).await;
914 }
915
916 for para in network_spec.parachains.iter() {
918 if para.registration_strategy() != &RegistrationStrategy::Manual {
919 warn!(
920 "⚠️ Parachain {} can't be registered automatically on a JAM chain, it needs to be registered manually.",
921 para.id
922 );
923 }
924 }
925
926 let jam_ctx = SpawnNodeCtx {
928 nodes_by_name,
929 ..jam_ctx
930 };
931
932 self.spawn_parachains(
933 &network_spec.parachains,
934 &jam_ctx,
935 &jam_global_files_to_inject,
936 spawn_concurrency,
937 &mut network,
938 )
939 .await?;
940
941 custom_processes::spawn_custom_processes(
942 &mut network,
943 network_spec,
944 ns.clone(),
945 configured_concurrency(network_spec),
946 )
947 .await?;
948
949 network.set_start_time_ts(start_time);
950
951 write_zombie_json(serde_json::to_value(&network)?, scoped_fs, ns.name()).await?;
952
953 if network_spec.global_settings.tear_down_on_failure() {
954 network.spawn_watching_task();
955 }
956
957 generators::cleanup_db_snapshot_cache(&resolved_db_snapshots).await;
958
959 Ok(network)
960 }
961}
962
963fn validate_provider_tag(
968 inner: &serde_json::Value,
969 node_name: &str,
970 provider_name: &str,
971) -> Result<(), OrchestratorError> {
972 let provider_tag = inner
973 .get("provider_tag")
974 .and_then(|v| v.as_str())
975 .ok_or_else(|| {
976 OrchestratorError::InvalidConfig(format!(
977 "Node '{node_name}' is missing `provider_tag` in inner node JSON"
978 ))
979 })?;
980
981 if provider_tag != provider_name {
982 return Err(OrchestratorError::InvalidConfigForProvider(
983 provider_name.to_string(),
984 provider_tag.to_string(),
985 ));
986 }
987
988 Ok(())
989}
990
991async fn recreate_network_nodes_from_json(
992 nodes_json: &serde_json::Value,
993 ns: DynNamespace,
994 provider_name: &str,
995) -> Result<Vec<Arc<NetworkNode>>, OrchestratorError> {
996 let raw_nodes: Vec<RawNetworkNode> = serde_json::from_value(nodes_json.clone())?;
997
998 let mut nodes = Vec::with_capacity(raw_nodes.len());
999 for raw in raw_nodes {
1000 validate_provider_tag(&raw.inner, &raw.name, provider_name)?;
1001
1002 let inner = ns.spawn_node_from_json(&raw.inner).await?;
1003 let relay_node = NetworkNode::new(
1004 raw.name,
1005 raw.ws_uri,
1006 raw.internal_ws_uri,
1007 raw.prometheus_uri,
1008 raw.multiaddr,
1009 raw.spec,
1010 inner,
1011 raw.cmd_generator_opts,
1012 raw.context,
1013 );
1014 nodes.push(Arc::new(relay_node));
1015 }
1016
1017 Ok(nodes)
1018}
1019
1020async fn recreate_relaychain_from_json(
1021 zombie_json: &serde_json::Value,
1022 ns: DynNamespace,
1023 provider_name: &str,
1024) -> Result<(Relaychain, NetworkSpec), OrchestratorError> {
1025 let relay_json = zombie_json
1026 .get("relay")
1027 .ok_or(OrchestratorError::InvalidConfig(
1028 "Missing `relay` field in zombie.json".into(),
1029 ))?
1030 .clone();
1031
1032 let mut relay_raw: RawRelaychain = serde_json::from_value(relay_json)?;
1033
1034 let initial_spec: NetworkSpec = serde_json::from_value(
1035 zombie_json
1036 .get("initial_spec")
1037 .ok_or(OrchestratorError::InvalidConfig(
1038 "Missing `initial_spec` field in zombie.json".into(),
1039 ))?
1040 .clone(),
1041 )?;
1042
1043 let nodes =
1045 recreate_network_nodes_from_json(&relay_raw.nodes, ns.clone(), provider_name).await?;
1046 relay_raw.inner.nodes = nodes;
1047
1048 Ok((relay_raw.inner, initial_spec))
1049}
1050
1051async fn recreate_jamchain_from_json(
1053 zombie_json: &serde_json::Value,
1054 ns: DynNamespace,
1055 provider_name: &str,
1056) -> Result<Option<Jamchain>, OrchestratorError> {
1057 let Some(jamchain_json) = zombie_json.get("jamchain") else {
1058 return Ok(None);
1059 };
1060
1061 let mut jamchain_raw: RawJamchain = serde_json::from_value(jamchain_json.clone())?;
1062 let raw_nodes: Vec<RawJamNetworkNode> = serde_json::from_value(jamchain_raw.nodes.clone())?;
1063
1064 let mut nodes = Vec::with_capacity(raw_nodes.len());
1065 for raw in raw_nodes {
1066 validate_provider_tag(&raw.inner, &raw.name, provider_name)?;
1067
1068 let inner = ns.spawn_node_from_json(&raw.inner).await?;
1069 nodes.push(Arc::new(JamNetworkNode::new(
1070 raw.name,
1071 inner,
1072 raw.spec,
1073 raw.ip,
1074 raw.cmd_generator_opts,
1075 )));
1076 }
1077
1078 jamchain_raw.inner.nodes = nodes;
1079
1080 Ok(Some(jamchain_raw.inner))
1081}
1082
1083async fn recreate_custom_processes_from_json(
1086 zombie_json: &serde_json::Value,
1087 ns: DynNamespace,
1088 provider_name: &str,
1089) -> Result<Vec<Arc<CustomProcessNode>>, OrchestratorError> {
1090 let Some(processes_json) = zombie_json.get("custom_processes") else {
1091 return Ok(vec![]);
1092 };
1093
1094 let raw_processes: Vec<RawCustomProcessNode> = serde_json::from_value(processes_json.clone())?;
1095
1096 let mut processes = Vec::with_capacity(raw_processes.len());
1097 for raw in raw_processes {
1098 validate_provider_tag(&raw.inner, &raw.name, provider_name)?;
1099
1100 let inner = match ns.spawn_node_from_json(&raw.inner).await {
1104 Ok(inner) => inner,
1105 Err(err) => {
1106 warn!(
1107 "⚠️ {}: could not attach to the custom process, skipping it: {err}",
1108 raw.name
1109 );
1110 continue;
1111 },
1112 };
1113
1114 let mut ports = raw.ports;
1118 let state = raw.state.unwrap_or(CustomProcessState::Ready);
1119 if !running_in_ci() && !raw.spec.one_shot() {
1122 for (port_name, port) in ports.iter_mut() {
1123 let saved = std::mem::take(&mut port.external);
1124 port.external =
1125 spawner::forward_or_keep(&inner, &raw.name, port_name, port.port, saved).await;
1126 }
1127 }
1128
1129 processes.push(Arc::new(CustomProcessNode::new(
1130 raw.name, inner, raw.spec, raw.ip, ports, state,
1131 )));
1132 }
1133
1134 Ok(processes)
1135}
1136
1137async fn recreate_parachains_from_json(
1138 zombie_json: &serde_json::Value,
1139 ns: DynNamespace,
1140 provider_name: &str,
1141) -> Result<HashMap<u32, Vec<Parachain>>, OrchestratorError> {
1142 let paras_json = zombie_json
1143 .get("parachains")
1144 .ok_or(OrchestratorError::InvalidConfig(
1145 "Missing `parachains` field in zombie.json".into(),
1146 ))?
1147 .clone();
1148
1149 let raw_paras: HashMap<u32, Vec<RawParachain>> = serde_json::from_value(paras_json)?;
1150
1151 let mut parachains_map = HashMap::new();
1152
1153 for (id, parachain_entries) in raw_paras {
1154 let mut parsed_vec = Vec::with_capacity(parachain_entries.len());
1155
1156 for raw_para in parachain_entries {
1157 let mut para = raw_para.inner;
1158 para.collators =
1159 recreate_network_nodes_from_json(&raw_para.collators, ns.clone(), provider_name)
1160 .await?;
1161 parsed_vec.push(para);
1162 }
1163
1164 parachains_map.insert(id, parsed_vec);
1165 }
1166
1167 Ok(parachains_map)
1168}
1169
1170fn split_nodes_by_bootnodes(
1173 nodes: &[NodeSpec],
1174 no_default_bootnodes: bool,
1175) -> (Vec<&NodeSpec>, Vec<&NodeSpec>) {
1176 let mut bootnodes = vec![];
1178 let mut other_nodes = vec![];
1179 nodes.iter().for_each(|node| {
1180 if node.is_bootnode {
1181 bootnodes.push(node)
1182 } else {
1183 other_nodes.push(node)
1184 }
1185 });
1186
1187 if bootnodes.is_empty() && !no_default_bootnodes {
1188 bootnodes.push(other_nodes.remove(0))
1189 }
1190
1191 (bootnodes, other_nodes)
1192}
1193
1194fn generate_bootnode_addr(
1196 node: &NetworkNode,
1197 ip: &IpAddr,
1198 port: u16,
1199) -> Result<String, GeneratorError> {
1200 generators::generate_node_bootnode_addr(
1201 &node.spec.peer_id,
1202 ip,
1203 port,
1204 node.args().as_ref(),
1205 &node.spec.p2p_cert_hash,
1206 )
1207}
1208
1209async fn upload_raw_spec(
1214 ns: &DynNamespace,
1215 chain_spec: &generators::chain_spec::ChainSpec,
1216) -> Result<(), OrchestratorError> {
1217 if let Some(location) = chain_spec.raw_path() {
1218 ns.upload_to_fileserver(&ns.base_dir().join(location))
1219 .await?;
1220 }
1221
1222 Ok(())
1223}
1224
1225fn validate_spec_with_provider_capabilities(
1227 network_spec: &NetworkSpec,
1228 capabilities: &ProviderCapabilities,
1229) -> Result<(), anyhow::Error> {
1230 let mut errs: Vec<String> = vec![];
1231
1232 for process in &network_spec.custom_processes {
1233 for var in process.env() {
1234 if !capabilities.has_secrets
1235 && matches!(var.value, configuration::EnvValue::Secret { .. })
1236 {
1237 errs.push(format!(
1238 "custom_processes['{}'].env: '{}' is read from a Secret, which only kubernetes has",
1239 process.name(),
1240 var.name
1241 ));
1242 }
1243 }
1244 for file in process.injected_files() {
1245 if NODE_MOUNTED_DIRS
1246 .iter()
1247 .any(|dir| Path::new(dir) == file.remote_path)
1248 {
1249 errs.push(format!(
1250 "custom_processes['{}'].injected_files: '{}' is a directory zombienet mounts",
1251 process.name(),
1252 file.remote_path.display()
1253 ));
1254 } else if let Some(shared) = [NODE_SCRIPTS_DIR, HELPERS_DIR]
1255 .iter()
1256 .find(|dir| file.remote_path.starts_with(dir))
1257 {
1258 errs.push(format!(
1260 "custom_processes['{}'].injected_files: '{}' is inside {shared}, which the containers of a network share",
1261 process.name(),
1262 file.remote_path.display()
1263 ));
1264 }
1265 if !file.local_path.is_file() {
1266 let from = if file.local_path.is_relative() {
1267 std::env::current_dir()
1268 .map(|cwd| format!(" (relative to {})", cwd.display()))
1269 .unwrap_or_default()
1270 } else {
1271 String::new()
1272 };
1273 errs.push(format!(
1274 "custom_processes['{}'].injected_files: no file at '{}'{from}",
1275 process.name(),
1276 file.local_path.display()
1277 ));
1278 }
1279 }
1280 }
1281
1282 if capabilities.requires_image {
1283 if let Some(relaychain) = network_spec.try_relaychain() {
1285 if relaychain.default_image.is_none() {
1286 let nodes = &relaychain.nodes;
1288 if nodes.iter().any(|node| node.image.is_none()) {
1289 errs.push(String::from(
1290 "Missing image for node, and not default is set at relaychain",
1291 ));
1292 }
1293 }
1294 };
1295
1296 for para in &network_spec.parachains {
1298 if para.default_image.is_none() {
1299 let nodes = ¶.collators;
1300 if nodes.iter().any(|node| node.image.is_none()) {
1301 errs.push(format!(
1302 "Missing image for node, and not default is set at parachain {}",
1303 para.id
1304 ));
1305 }
1306 }
1307 }
1308 } else {
1309 let mut cmds: HashSet<&str> = Default::default();
1312 if let Some(relaychain) = network_spec.try_relaychain() {
1313 if let Some(cmd) = relaychain.default_command.as_ref() {
1314 cmds.insert(cmd.as_str());
1315 }
1316 for node in relaychain.nodes.iter() {
1317 cmds.insert(node.command());
1318 }
1319 }
1320
1321 if let Some(jamchain) = network_spec.jamchain.as_ref() {
1322 for node in jamchain.nodes.iter() {
1323 cmds.insert(node.command());
1324 }
1325 }
1326
1327 for para in &network_spec.parachains {
1329 if let Some(cmd) = para.default_command.as_ref() {
1330 cmds.insert(cmd.as_str());
1331 }
1332
1333 for node in para.collators.iter() {
1334 cmds.insert(node.command());
1335 }
1336 }
1337
1338 let path = std::env::var("PATH").unwrap_or_default(); trace!("current PATH: {path}");
1341 let parts: Vec<_> = path.split(":").collect();
1342 for cmd in cmds {
1343 let missing = if cmd.contains('/') {
1344 trace!("checking {cmd}");
1345 if std::fs::metadata(cmd).is_err() {
1346 true
1347 } else {
1348 info!("🔎 We will use the full path {cmd} to spawn nodes.");
1349 false
1350 }
1351 } else {
1352 !parts.iter().any(|part| {
1354 let path_to = format!("{part}/{cmd}");
1355 trace!("checking {path_to}");
1356 let check_result = std::fs::metadata(&path_to);
1357 trace!("result {:?}", check_result);
1358 if check_result.is_ok() {
1359 info!("🔎 We will use the cmd: '{cmd}' at path {path_to} to spawn nodes.");
1360 true
1361 } else {
1362 false
1363 }
1364 })
1365 };
1366
1367 if missing {
1368 errs.push(help_msg(cmd));
1369 }
1370 }
1371 }
1372
1373 if !errs.is_empty() {
1374 let msg = errs.join("\n");
1375 return Err(anyhow::anyhow!(format!("Invalid configuration: \n {msg}")));
1376 }
1377
1378 Ok(())
1379}
1380
1381fn help_msg(cmd: &str) -> String {
1382 match cmd {
1383 "parachain-template-node" | "solochain-template-node" | "minimal-template-node" => {
1384 format!("Missing binary {cmd}, compile by running: \n\tcargo build --package {cmd} --release")
1385 },
1386 "polkadot" => {
1387 format!("Missing binary {cmd}, compile by running (in the polkadot-sdk repo): \n\t cargo build --locked --release --features fast-runtime --bin {cmd} --bin polkadot-prepare-worker --bin polkadot-execute-worker")
1388 },
1389 "polkadot-parachain" => {
1390 format!("Missing binary {cmd}, compile by running (in the polkadot-sdk repo): \n\t cargo build --release --locked -p {cmd}-bin --bin {cmd}")
1391 },
1392 _ => {
1393 format!("Missing binary {cmd}, please compile it.")
1394 },
1395 }
1396}
1397
1398fn spawn_concurrency_from_env() -> Option<usize> {
1400 if let Ok(concurrency) = env::var("ZOMBIE_SPAWN_CONCURRENCY") {
1401 concurrency.parse::<usize>().ok()
1402 } else {
1403 None
1404 }
1405}
1406
1407fn configured_concurrency(spec: &NetworkSpec) -> usize {
1412 spawn_concurrency_from_env()
1413 .or(spec.global_settings.spawn_concurrency())
1414 .unwrap_or(20)
1415 .max(1)
1416}
1417
1418fn chains_have_tokens(spec: &NetworkSpec) -> Result<bool, anyhow::Error> {
1422 let chains = (&spec.relaychain, &spec.jamchain, &spec.parachains);
1423 Ok(has_tokens(&serde_json::to_string(&chains)?))
1424}
1425
1426fn calculate_concurrency(spec: &NetworkSpec) -> Result<(usize, bool), anyhow::Error> {
1428 let concurrency = configured_concurrency(spec);
1429 if concurrency > 1 && chains_have_tokens(spec)? {
1430 Ok((1, true))
1431 } else {
1432 Ok((concurrency, false))
1433 }
1434}
1435
1436fn dependency_levels_among<'a>(
1441 nodes: &'a [&'a NodeSpec],
1442) -> Result<Vec<Vec<&'a NodeSpec>>, OrchestratorError> {
1443 let by_name = nodes
1444 .iter()
1445 .map(|n| (n.name.as_str(), *n))
1446 .collect::<HashMap<_, _>>();
1447
1448 let mut graph = HashMap::with_capacity(nodes.len());
1449 let mut indegree = HashMap::with_capacity(nodes.len());
1450
1451 for node in nodes {
1452 graph.insert(node.name.as_str(), Vec::new());
1453 indegree.insert(node.name.as_str(), 0);
1454 }
1455
1456 for &node in nodes {
1458 if let Ok(args_json) = serde_json::to_string(&node.args) {
1459 let unique_deps = get_tokens_to_replace(&args_json)
1461 .into_iter()
1462 .filter(|dep| dep != &node.name)
1463 .filter_map(|dep| by_name.get(dep.as_str()))
1464 .map(|&dep_node| dep_node.name.as_str())
1465 .collect::<HashSet<_>>();
1466
1467 for dep_name in unique_deps {
1468 graph
1469 .get_mut(dep_name)
1470 .expect(&format!("{GRAPH_CONTAINS_DEP} {THIS_IS_A_BUG}"))
1471 .push(node);
1472 *indegree
1473 .get_mut(node.name.as_str())
1474 .expect(&format!("{INDEGREE_CONTAINS_NAME} {THIS_IS_A_BUG}")) += 1;
1475 }
1476 }
1477 }
1478
1479 let mut queue = nodes
1481 .iter()
1482 .filter(|n| {
1483 *indegree
1484 .get(n.name.as_str())
1485 .expect(&format!("{INDEGREE_CONTAINS_NAME} {THIS_IS_A_BUG}"))
1486 == 0
1487 })
1488 .copied()
1489 .collect::<VecDeque<_>>();
1490
1491 let mut processed_count = 0;
1492 let mut levels = Vec::new();
1493
1494 while !queue.is_empty() {
1496 let level_size = queue.len();
1497 let mut current_level = Vec::with_capacity(level_size);
1498
1499 for _ in 0..level_size {
1500 let n = queue
1501 .pop_front()
1502 .expect(&format!("{QUEUE_NOT_EMPTY} {THIS_IS_A_BUG}"));
1503 current_level.push(n);
1504 processed_count += 1;
1505
1506 for &neighbour in graph
1507 .get(n.name.as_str())
1508 .expect(&format!("{GRAPH_CONTAINS_NAME} {THIS_IS_A_BUG}"))
1509 {
1510 let neighbour_indegree = indegree
1511 .get_mut(neighbour.name.as_str())
1512 .expect(&format!("{INDEGREE_CONTAINS_NAME} {THIS_IS_A_BUG}"));
1513 *neighbour_indegree -= 1;
1514
1515 if *neighbour_indegree == 0 {
1516 queue.push_back(neighbour);
1517 }
1518 }
1519 }
1520
1521 current_level.sort_by_key(|n| &n.name);
1522 levels.push(current_level);
1523 }
1524
1525 if processed_count != nodes.len() {
1527 return Err(OrchestratorError::InvalidConfig(
1528 "Tokens have cyclical dependencies".to_string(),
1529 ));
1530 }
1531
1532 Ok(levels)
1533}
1534
1535#[derive(Clone, Debug)]
1543pub struct ScopedFilesystem<'a, FS: FileSystem> {
1544 fs: &'a FS,
1545 base_dir: &'a str,
1546}
1547
1548impl<'a, FS: FileSystem> ScopedFilesystem<'a, FS> {
1549 pub fn new(fs: &'a FS, base_dir: &'a str) -> Self {
1550 Self { fs, base_dir }
1551 }
1552
1553 async fn copy_files(&self, files: Vec<&TransferedFile>) -> Result<(), FileSystemError> {
1554 for file in files {
1555 let full_remote_path = PathBuf::from(format!(
1556 "{}/{}",
1557 self.base_dir,
1558 file.remote_path.to_string_lossy()
1559 ));
1560 trace!("coping file: {file}");
1561 self.fs
1562 .copy(file.local_path.as_path(), full_remote_path)
1563 .await?;
1564 }
1565 Ok(())
1566 }
1567
1568 async fn read(&self, file: impl AsRef<Path>) -> Result<Vec<u8>, FileSystemError> {
1569 let file = file.as_ref();
1570
1571 let full_path = if file.is_absolute() {
1572 file.to_owned()
1573 } else {
1574 PathBuf::from(format!("{}/{}", self.base_dir, file.to_string_lossy()))
1575 };
1576 let content = self.fs.read(full_path).await?;
1577 Ok(content)
1578 }
1579
1580 async fn read_to_string(&self, file: impl AsRef<Path>) -> Result<String, FileSystemError> {
1581 let file = file.as_ref();
1582
1583 let full_path = if file.is_absolute() {
1584 file.to_owned()
1585 } else {
1586 PathBuf::from(format!("{}/{}", self.base_dir, file.to_string_lossy()))
1587 };
1588 let content = self.fs.read_to_string(full_path).await?;
1589 Ok(content)
1590 }
1591
1592 async fn create_dir(&self, path: impl AsRef<Path>) -> Result<(), FileSystemError> {
1593 let path = PathBuf::from(format!(
1594 "{}/{}",
1595 self.base_dir,
1596 path.as_ref().to_string_lossy()
1597 ));
1598 self.fs.create_dir(path).await
1599 }
1600
1601 async fn create_dir_all(&self, path: impl AsRef<Path>) -> Result<(), FileSystemError> {
1602 let path = PathBuf::from(format!(
1603 "{}/{}",
1604 self.base_dir,
1605 path.as_ref().to_string_lossy()
1606 ));
1607 self.fs.create_dir_all(path).await
1608 }
1609
1610 async fn write(
1611 &self,
1612 path: impl AsRef<Path>,
1613 contents: impl AsRef<[u8]> + Send,
1614 ) -> Result<(), FileSystemError> {
1615 let path = path.as_ref();
1616
1617 let full_path = if path.is_absolute() {
1618 path.to_owned()
1619 } else {
1620 PathBuf::from(format!("{}/{}", self.base_dir, path.to_string_lossy()))
1621 };
1622
1623 self.fs.write(full_path, contents).await
1624 }
1625
1626 fn full_path(&self, path: impl AsRef<Path>) -> PathBuf {
1628 let path = path.as_ref();
1629
1630 let full_path = if path.is_absolute() {
1631 path.to_owned()
1632 } else {
1633 PathBuf::from(format!("{}/{}", self.base_dir, path.to_string_lossy()))
1634 };
1635
1636 full_path
1637 }
1638
1639 fn base_dir(&self) -> &str {
1641 self.base_dir
1642 }
1643}
1644
1645#[derive(Clone, Debug)]
1646pub enum ZombieRole {
1647 Temp,
1648 Node,
1649 Bootnode,
1650 Collator,
1651 CumulusCollator,
1652}
1653
1654impl ZombieRole {
1655 pub(crate) fn node_role(&self) -> NodeRole {
1656 match self {
1657 ZombieRole::Node => NodeRole::Node,
1658 ZombieRole::Bootnode => NodeRole::Bootnode,
1659 ZombieRole::Collator => NodeRole::Collator,
1660 ZombieRole::CumulusCollator => NodeRole::CumulusCollator,
1661 ZombieRole::Temp => NodeRole::Temp,
1662 }
1663 }
1664}
1665
1666pub use network::{AddCollatorOptions, AddNodeOptions};
1668pub use network_helper::metrics;
1669pub use sc_chain_spec;
1670
1671#[cfg(test)]
1672mod tests {
1673 use configuration::{GlobalSettingsBuilder, NetworkConfigBuilder};
1674 use lazy_static::lazy_static;
1675 use tokio::sync::Mutex;
1676
1677 use super::*;
1678
1679 const ENV_KEY: &str = "ZOMBIE_SPAWN_CONCURRENCY";
1680 lazy_static! {
1682 static ref ENV_MUTEX: Mutex<()> = Mutex::new(());
1683 }
1684
1685 fn set_env(concurrency: Option<u32>) {
1686 if let Some(value) = concurrency {
1687 env::set_var(ENV_KEY, value.to_string());
1688 } else {
1689 env::remove_var(ENV_KEY);
1690 }
1691 }
1692
1693 fn generate(
1694 with_image: bool,
1695 with_cmd: Option<&'static str>,
1696 ) -> Result<NetworkConfig, Vec<anyhow::Error>> {
1697 NetworkConfigBuilder::new()
1698 .with_relaychain(|r| {
1699 let mut relay = r
1700 .with_chain("rococo-local")
1701 .with_default_command(with_cmd.unwrap_or("polkadot"));
1702 if with_image {
1703 relay = relay.with_default_image("docker.io/parity/polkadot")
1704 }
1705
1706 relay
1707 .with_validator(|node| node.with_name("alice"))
1708 .with_validator(|node| node.with_name("bob"))
1709 })
1710 .with_parachain(|p| {
1711 p.with_id(2000).cumulus_based(true).with_collator(|n| {
1712 let node = n
1713 .with_name("collator")
1714 .with_command(with_cmd.unwrap_or("polkadot-parachain"));
1715 if with_image {
1716 node.with_image("docker.io/paritypr/test-parachain")
1717 } else {
1718 node
1719 }
1720 })
1721 })
1722 .build()
1723 }
1724
1725 fn get_node_with_dependencies(name: &str, dependencies: Option<Vec<&NodeSpec>>) -> NodeSpec {
1726 let mut spec = NodeSpec {
1727 name: name.to_string(),
1728 ..Default::default()
1729 };
1730 if let Some(dependencies) = dependencies {
1731 for node in dependencies {
1732 spec.args.push(
1733 format!("{{{{ZOMBIE:{}:someField}}}}", node.name)
1734 .as_str()
1735 .into(),
1736 );
1737 }
1738 }
1739 spec
1740 }
1741
1742 fn verify_levels(actual_levels: Vec<Vec<&NodeSpec>>, expected_levels: Vec<Vec<&str>>) {
1743 actual_levels
1744 .iter()
1745 .zip(expected_levels)
1746 .for_each(|(actual_level, expected_level)| {
1747 assert_eq!(actual_level.len(), expected_level.len());
1748 actual_level
1749 .iter()
1750 .zip(expected_level.iter())
1751 .for_each(|(node, expected_name)| assert_eq!(node.name, *expected_name));
1752 });
1753 }
1754
1755 #[tokio::test]
1756 async fn valid_config_with_image() {
1757 let network_config = generate(true, None).unwrap();
1758 let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1759 let caps = ProviderCapabilities {
1760 requires_image: true,
1761 has_resources: false,
1762 prefix_with_full_path: false,
1763 use_default_ports_in_cmd: false,
1764 has_secrets: false,
1765 fileserver_fetches_db_snapshot_url: false,
1766 };
1767
1768 let valid = validate_spec_with_provider_capabilities(&spec, &caps);
1769 assert!(valid.is_ok())
1770 }
1771
1772 #[tokio::test]
1773 async fn invalid_config_without_image() {
1774 let network_config = generate(false, None).unwrap();
1775 let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1776 let caps = ProviderCapabilities {
1777 requires_image: true,
1778 has_resources: false,
1779 prefix_with_full_path: false,
1780 use_default_ports_in_cmd: false,
1781 has_secrets: false,
1782 fileserver_fetches_db_snapshot_url: false,
1783 };
1784
1785 let valid = validate_spec_with_provider_capabilities(&spec, &caps);
1786 assert!(valid.is_err())
1787 }
1788
1789 #[tokio::test]
1790 async fn invalid_config_missing_cmd() {
1791 let network_config = generate(false, Some("other")).unwrap();
1792 let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1793 let caps = ProviderCapabilities {
1794 requires_image: false,
1795 has_resources: false,
1796 prefix_with_full_path: false,
1797 use_default_ports_in_cmd: false,
1798 has_secrets: false,
1799 fileserver_fetches_db_snapshot_url: false,
1800 };
1801
1802 let valid = validate_spec_with_provider_capabilities(&spec, &caps);
1803 assert!(valid.is_err())
1804 }
1805
1806 #[tokio::test]
1807 async fn valid_config_present_cmd() {
1808 let network_config = generate(false, Some("cargo")).unwrap();
1809 let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1810 let caps = ProviderCapabilities {
1811 requires_image: false,
1812 has_resources: false,
1813 prefix_with_full_path: false,
1814 use_default_ports_in_cmd: false,
1815 has_secrets: false,
1816 fileserver_fetches_db_snapshot_url: false,
1817 };
1818
1819 let valid = validate_spec_with_provider_capabilities(&spec, &caps);
1820 println!("{valid:?}");
1821 assert!(valid.is_ok())
1822 }
1823
1824 fn image_caps() -> ProviderCapabilities {
1826 ProviderCapabilities {
1827 requires_image: true,
1828 has_resources: false,
1829 prefix_with_full_path: false,
1830 use_default_ports_in_cmd: true,
1831 has_secrets: false,
1832 fileserver_fetches_db_snapshot_url: false,
1833 }
1834 }
1835
1836 async fn spec_with_file(local: &str, remote: &str) -> NetworkSpec {
1838 let network_config = generate(true, None).unwrap();
1839 let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1840 spec.custom_processes.push(
1841 configuration::CustomProcessBuilder::new()
1842 .with_name("api")
1843 .with_command("api")
1844 .with_injected_file(configuration::TransferedFile::new(local, remote))
1845 .build()
1846 .unwrap(),
1847 );
1848 spec
1849 }
1850
1851 const AN_EXISTING_FILE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml");
1852
1853 #[tokio::test]
1854 async fn secret_env_needs_a_provider_with_secrets() {
1855 let network_config = generate(true, None).unwrap();
1856 let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1857 spec.custom_processes.push(
1858 configuration::CustomProcessBuilder::new()
1859 .with_name("api")
1860 .with_command("api")
1861 .with_env(vec![configuration::ProcessEnvVar::from_secret(
1862 "TURN_SECRET",
1863 "turn",
1864 "secret",
1865 )])
1866 .build()
1867 .unwrap(),
1868 );
1869
1870 let err = validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap_err();
1871 assert!(
1872 err.to_string()
1873 .contains("custom_processes['api'].env: 'TURN_SECRET' is read from a Secret"),
1874 "{err}"
1875 );
1876 let with = ProviderCapabilities {
1877 has_secrets: true,
1878 ..image_caps()
1879 };
1880 validate_spec_with_provider_capabilities(&spec, &with).unwrap();
1881 }
1882
1883 #[tokio::test]
1884 async fn injected_files_must_exist_before_anything_spawns() {
1885 let spec = spec_with_file("./nowhere/init.sql", "/init.sql").await;
1886 let err = validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap_err();
1887 assert!(
1888 err.to_string().contains(
1889 "custom_processes['api'].injected_files: no file at './nowhere/init.sql' (relative to"
1890 ),
1891 "{err}"
1892 );
1893
1894 let spec = spec_with_file(AN_EXISTING_FILE, "/init.sql").await;
1895 validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap();
1896 }
1897
1898 #[tokio::test]
1899 async fn injected_files_may_not_replace_a_mounted_directory() {
1900 let spec = spec_with_file(AN_EXISTING_FILE, "/data").await;
1901 let err = validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap_err();
1902 assert!(
1903 err.to_string().ends_with(
1904 "custom_processes['api'].injected_files: '/data' is a directory zombienet mounts"
1905 ),
1906 "{err}"
1907 );
1908
1909 let spec = spec_with_file(AN_EXISTING_FILE, "/data/init.sql").await;
1910 validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap();
1911
1912 let spec = spec_with_file(AN_EXISTING_FILE, "/scripts/run.sh").await;
1913 let err = validate_spec_with_provider_capabilities(&spec, &image_caps()).unwrap_err();
1914 assert!(
1915 err.to_string().ends_with(
1916 "custom_processes['api'].injected_files: '/scripts/run.sh' is inside /scripts, which the containers of a network share"
1917 ),
1918 "{err}"
1919 );
1920 }
1921
1922 #[tokio::test]
1923 async fn default_spawn_concurrency() {
1924 let _g = ENV_MUTEX.lock().await;
1925 set_env(None);
1926 let network_config = generate(false, Some("cargo")).unwrap();
1927 let spec = NetworkSpec::from_config(&network_config).await.unwrap();
1928 let (concurrency, _) = calculate_concurrency(&spec).unwrap();
1929 assert_eq!(concurrency, 20);
1930 }
1931
1932 #[tokio::test]
1933 async fn set_spawn_concurrency() {
1934 let _g = ENV_MUTEX.lock().await;
1935 set_env(None);
1936
1937 let network_config = generate(false, Some("cargo")).unwrap();
1938 let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1939
1940 let global_settings = GlobalSettingsBuilder::new()
1941 .with_spawn_concurrency(4)
1942 .build()
1943 .unwrap();
1944
1945 spec.set_global_settings(global_settings);
1946 let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
1947 assert_eq!(concurrency, 4);
1948 assert!(!limited);
1949 }
1950
1951 #[tokio::test]
1952 async fn set_spawn_concurrency_but_limited() {
1953 let _g = ENV_MUTEX.lock().await;
1954 set_env(None);
1955
1956 let network_config = generate(false, Some("cargo")).unwrap();
1957 let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1958
1959 let global_settings = GlobalSettingsBuilder::new()
1960 .with_spawn_concurrency(4)
1961 .build()
1962 .unwrap();
1963
1964 spec.set_global_settings(global_settings);
1965 let node = spec.relaychain_mut().nodes.first_mut().unwrap();
1966 node.args
1967 .push("--bootnodes {{ZOMBIE:bob:multiAddress')}}".into());
1968 let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
1969 assert_eq!(concurrency, 1);
1970 assert!(limited);
1971 }
1972
1973 #[tokio::test]
1974 async fn custom_process_placeholders_do_not_limit_spawn_concurrency() {
1975 let _g = ENV_MUTEX.lock().await;
1976 set_env(None);
1977
1978 let network_config = generate(false, Some("cargo")).unwrap();
1979 let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
1980
1981 let global_settings = GlobalSettingsBuilder::new()
1982 .with_spawn_concurrency(4)
1983 .build()
1984 .unwrap();
1985
1986 spec.set_global_settings(global_settings);
1987 spec.custom_processes.push(
1988 configuration::CustomProcessBuilder::new()
1989 .with_name("api")
1990 .with_command("api")
1991 .with_args(vec!["{{ZOMBIE:alice:ws_uri}}".into()])
1992 .build()
1993 .unwrap(),
1994 );
1995 let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
1996 assert_eq!(concurrency, 4);
1997 assert!(!limited);
1998 }
1999
2000 #[tokio::test]
2001 async fn set_spawn_concurrency_from_env() {
2002 let _g = ENV_MUTEX.lock().await;
2003 set_env(Some(10));
2004
2005 let network_config = generate(false, Some("cargo")).unwrap();
2006 let spec = NetworkSpec::from_config(&network_config).await.unwrap();
2007 let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
2008 assert_eq!(concurrency, 10);
2009 assert!(!limited);
2010 }
2011
2012 #[tokio::test]
2013 async fn set_spawn_concurrency_from_env_but_limited() {
2014 let _g = ENV_MUTEX.lock().await;
2015 set_env(Some(12));
2016
2017 let network_config = generate(false, Some("cargo")).unwrap();
2018 let mut spec = NetworkSpec::from_config(&network_config).await.unwrap();
2019 let node = spec.relaychain_mut().nodes.first_mut().unwrap();
2020 node.args
2021 .push("--bootnodes {{ZOMBIE:bob:multiAddress')}}".into());
2022 let (concurrency, limited) = calculate_concurrency(&spec).unwrap();
2023 assert_eq!(concurrency, 1);
2024 assert!(limited);
2025 }
2026
2027 #[test]
2028 fn dependency_levels_among_should_work() {
2029 assert!(dependency_levels_among(&[]).unwrap().is_empty());
2031
2032 let alice = get_node_with_dependencies("alice", None);
2034 let nodes = [&alice];
2035
2036 let levels = dependency_levels_among(&nodes).unwrap();
2037 let expected = vec![vec!["alice"]];
2038
2039 verify_levels(levels, expected);
2040
2041 let alice = get_node_with_dependencies("alice", None);
2043 let bob = get_node_with_dependencies("bob", None);
2044 let nodes = [&alice, &bob];
2045
2046 let levels = dependency_levels_among(&nodes).unwrap();
2047 let expected = vec![vec!["alice", "bob"]];
2048
2049 verify_levels(levels, expected);
2050
2051 let alice = get_node_with_dependencies("alice", None);
2053 let bob = get_node_with_dependencies("bob", Some(vec![&alice]));
2054 let charlie = get_node_with_dependencies("charlie", Some(vec![&bob]));
2055 let nodes = [&alice, &bob, &charlie];
2056
2057 let levels = dependency_levels_among(&nodes).unwrap();
2058 let expected = vec![vec!["alice"], vec!["bob"], vec!["charlie"]];
2059
2060 verify_levels(levels, expected);
2061
2062 let alice = get_node_with_dependencies("alice", None);
2066 let bob = get_node_with_dependencies("bob", Some(vec![&alice]));
2067 let charlie = get_node_with_dependencies("charlie", Some(vec![&alice]));
2068 let nodes = [&alice, &bob, &charlie];
2069
2070 let levels = dependency_levels_among(&nodes).unwrap();
2071 let expected = vec![vec!["alice"], vec!["bob", "charlie"]];
2072
2073 verify_levels(levels, expected);
2074
2075 let alice = get_node_with_dependencies("alice", None);
2079 let bob = get_node_with_dependencies("bob", Some(vec![&alice]));
2080 let charlie = get_node_with_dependencies("charlie", Some(vec![&alice]));
2081 let dave = get_node_with_dependencies("dave", Some(vec![&charlie, &bob]));
2082 let nodes = [&alice, &bob, &charlie, &dave];
2083
2084 let levels = dependency_levels_among(&nodes).unwrap();
2085 let expected = vec![vec!["alice"], vec!["bob", "charlie"], vec!["dave"]];
2086
2087 verify_levels(levels, expected);
2088 }
2089
2090 #[test]
2091 fn dependency_levels_among_should_detect_cycles() {
2092 let mut alice = get_node_with_dependencies("alice", None);
2093 let bob = get_node_with_dependencies("bob", Some(vec![&alice]));
2094 alice.args.push("{{ZOMBIE:bob:someField}}".into());
2095
2096 assert!(dependency_levels_among(&[&alice, &bob]).is_err())
2097 }
2098}