try_runtime_core/common/
state.rs

1// This file is part of try-runtime-cli.
2
3// Copyright (C) Parity Technologies (UK) Ltd.
4// SPDX-License-Identifier: Apache-2.0
5
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9//
10// 	http://www.apache.org/licenses/LICENSE-2.0
11//
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17
18use std::{fmt::Debug, path::PathBuf, str::FromStr};
19
20use frame_remote_externalities::{
21    Builder, Mode, OfflineConfig, OnlineConfig, RemoteExternalities, SnapshotConfig,
22};
23use parity_scale_codec::Decode;
24use sc_cli::{execution_method_from_cli, RuntimeVersion};
25use sc_executor::{
26    sp_wasm_interface::HostFunctions, HeapAllocStrategy, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY,
27};
28use sp_api::{CallContext, StorageProof};
29use sp_core::{
30    hexdisplay::HexDisplay, storage::well_known_keys, traits::ReadRuntimeVersion, twox_128, Hasher,
31};
32use sp_externalities::Extensions;
33use sp_runtime::{
34    traits::{BlakeTwo256, Block as BlockT, HashingFor, Header as HeaderT},
35    DeserializeOwned,
36};
37use sp_state_machine::{OverlayedChanges, StateMachine, TestExternalities, TrieBackendBuilder};
38use substrate_rpc_client::{ws_client, ChainApi};
39
40use crate::{
41    common::{
42        parse,
43        shared_parameters::{Runtime, SharedParams},
44    },
45    hash_of, rpc_err_handler, LOG_TARGET,
46};
47
48/// A `Live` variant for [`State`]
49#[derive(Debug, Clone, clap::Args)]
50pub struct LiveState {
51    /// The url(s) to connect to. Can be provided multiple times for parallel state download.
52    #[arg(
53		short,
54		long,
55		value_parser = parse::url,
56		num_args = 1..,
57		required = true,
58	)]
59    pub uri: Vec<String>,
60
61    /// The block hash at which to fetch the state.
62    ///
63    /// If non provided, then the latest finalized head is used.
64    #[arg(
65		short,
66		long,
67		value_parser = parse::hash,
68	)]
69    pub at: Option<String>,
70
71    /// A pallet to scrape. Can be provided multiple times. If empty, entire chain state will
72    /// be scraped.
73    ///
74    /// This is equivalent to passing `xx_hash_64(pallet)` to `--hashed_prefixes`.
75    #[arg(short, long, num_args = 1..)]
76    pub pallet: Vec<String>,
77
78    /// Storage entry key prefixes to scrape and inject into the test externalities. Pass as 0x
79    /// prefixed hex strings. By default, all keys are scraped and included.
80    #[arg(long = "prefix", value_parser = parse::hash, num_args = 1..)]
81    pub hashed_prefixes: Vec<String>,
82
83    /// Fetch the child-keys as well.
84    ///
85    /// Default is `false`, if specific `--pallets` are specified, `true` otherwise. In other
86    /// words, if you scrape the whole state the child tree data is included out of the box.
87    /// Otherwise, it must be enabled explicitly using this flag.
88    #[arg(long)]
89    pub child_tree: bool,
90}
91
92impl LiveState {
93    /// Return the `at` block hash as a `Hash`, if it exists.
94    pub fn at<Block: BlockT>(&self) -> sc_cli::Result<Option<<Block>::Hash>>
95    where
96        <Block::Hash as FromStr>::Err: Debug,
97    {
98        self.at
99            .clone()
100            .map(|s| hash_of::<Block>(s.as_str()))
101            .transpose()
102    }
103
104    /// Converts this `LiveState` into a `LiveState` for the previous block.
105    ///
106    /// Useful for opertations like when you want to execute a block, but also need the state of the
107    /// block *before* it.
108    pub async fn to_prev_block_live_state<Block: BlockT>(self) -> sc_cli::Result<LiveState>
109    where
110        <Block::Hash as FromStr>::Err: Debug,
111    {
112        // We want to execute the block `at`, therefore need the state of the block *before* it.
113        let at = self.at::<Block>()?;
114
115        // Get the block number requested by the user, or the current block number if they
116        // didn't specify one.
117        let rpc = ws_client(&self.uri[0]).await?;
118        let previous_hash = ChainApi::<(), Block::Hash, Block::Header, ()>::header(&rpc, at)
119            .await
120            .map_err(rpc_err_handler)
121            .and_then(|maybe_header| {
122                maybe_header
123                    .ok_or("header_not_found")
124                    .map(|h| *h.parent_hash())
125            })?;
126
127        Ok(LiveState {
128            at: Some(hex::encode(previous_hash)),
129            ..self
130        })
131    }
132}
133
134/// The source of runtime *state* to use.
135#[derive(Debug, Clone, clap::Subcommand)]
136pub enum State {
137    /// Use a state snapshot as the source of runtime state.
138    Snap {
139        #[clap(short = 'p', long = "path", alias = "snapshot-path")]
140        path: Option<PathBuf>,
141    },
142
143    /// Use a live chain as the source of runtime state.
144    Live(LiveState),
145}
146
147/// Checks to perform on the given runtime, compared to the existing runtime.
148#[derive(Debug, Clone, Copy)]
149pub struct RuntimeChecks {
150    /// Enforce the `spec_name`s match
151    pub name_matches: bool,
152    /// Enforce the `spec_version` of the given is greater or equal to the existing
153    /// runtime.
154    pub version_increases: bool,
155    /// Enforce that the given runtime is compiled with the try-runtime feature.
156    pub try_runtime_feature_enabled: bool,
157}
158
159impl State {
160    /// Create the [`RemoteExternalities`].
161    ///
162    /// This will override the code as it sees fit based on [`Runtime`]. It will also check the
163    /// spec-version and name.
164    pub async fn to_ext<Block: BlockT + DeserializeOwned, HostFns: HostFunctions>(
165        &self,
166        shared: &SharedParams,
167        executor: &WasmExecutor<HostFns>,
168        state_snapshot: Option<SnapshotConfig>,
169        runtime_checks: RuntimeChecks,
170    ) -> sc_cli::Result<RemoteExternalities<Block>>
171    where
172        Block::Header: DeserializeOwned,
173        <Block::Hash as FromStr>::Err: Debug,
174    {
175        let builder = match self {
176            State::Snap { path } => {
177                let path = path
178                    .as_ref()
179                    .ok_or_else(|| "no snapshot path provided".to_string())?;
180
181                Builder::<Block>::new().mode(Mode::Offline(OfflineConfig {
182                    state_snapshot: SnapshotConfig::new(path),
183                }))
184            }
185            State::Live(LiveState {
186                pallet,
187                uri,
188                at,
189                child_tree,
190                hashed_prefixes,
191            }) => {
192                let at = match at {
193                    Some(at_str) => Some(hash_of::<Block>(at_str)?),
194                    None => None,
195                };
196                let hashed_prefixes = hashed_prefixes
197                    .iter()
198                    .map(|p_str| {
199                        hex::decode(p_str).map_err(|e| {
200                            format!(
201                                "Error decoding `hashed_prefixes` hex string entry '{:?}' to bytes: {:?}",
202                                p_str, e
203                            )
204                        })
205                    })
206                    .collect::<Result<Vec<_>, _>>()?;
207                Builder::<Block>::new().mode(Mode::Online(OnlineConfig {
208                    at,
209                    transport_uris: uri.clone(),
210                    state_snapshot,
211                    pallets: pallet.clone(),
212                    child_trie: *child_tree,
213                    hashed_keys: vec![
214                        // we always download the code, but we almost always won't use it, based on
215                        // `Runtime`.
216                        well_known_keys::CODE.to_vec(),
217                        // we will always download this key, since it helps detect if we should do
218                        // runtime migration or not.
219                        [twox_128(b"System"), twox_128(b"LastRuntimeUpgrade")].concat(),
220                        [twox_128(b"System"), twox_128(b"Number")].concat(),
221                    ],
222                    hashed_prefixes,
223                }))
224            }
225        };
226
227        // possibly overwrite the state version, should hardly be needed.
228        let builder = if let Some(state_version) = shared.overwrite_state_version {
229            log::warn!(
230                target: LOG_TARGET,
231                "overwriting state version to {:?}, you better know what you are doing.",
232                state_version
233            );
234            builder.overwrite_state_version(state_version)
235        } else {
236            builder
237        };
238
239        // then, we prepare to replace the code based on what the CLI wishes.
240        let maybe_code_to_overwrite = match shared.runtime {
241            Runtime::Path(ref path) => Some(std::fs::read(path).map_err(|e| {
242                format!("error while reading runtime file from {:?}: {:?}", path, e)
243            })?),
244            Runtime::Existing => None,
245        };
246
247        // build the main ext.
248        let mut ext = builder.build().await?;
249
250        // actually replace the code if needed.
251        if let Some(new_code) = maybe_code_to_overwrite {
252            let original_code = ext
253                .execute_with(|| sp_io::storage::get(well_known_keys::CODE))
254                .expect("':CODE:' is always downloaded in try-runtime-cli; qed");
255
256            // NOTE: see the impl notes of `read_runtime_version`, the ext is almost not used here,
257            // only as a backup.
258            ext.insert(well_known_keys::CODE.to_vec(), new_code.clone());
259            let old_version = <RuntimeVersion as Decode>::decode(
260                &mut &*executor
261                    .read_runtime_version(&original_code, &mut ext.ext())
262                    .unwrap(),
263            )
264            .unwrap();
265            let old_code_hash =
266                HexDisplay::from(BlakeTwo256::hash(&original_code).as_fixed_bytes()).to_string();
267            log::info!(
268                target: LOG_TARGET,
269                "Original runtime [Name: {:?}] [Version: {:?}] [Code hash: 0x{}...{}]",
270                old_version.spec_name,
271                old_version.spec_version,
272                &old_code_hash[0..4],
273                &old_code_hash[old_code_hash.len() - 4..],
274            );
275            log::debug!(
276                target: LOG_TARGET,
277                "Original runtime full code hash: 0x{:?}",
278                old_code_hash,
279            );
280            let new_version = <RuntimeVersion as Decode>::decode(
281                &mut &*executor
282                    .read_runtime_version(&new_code, &mut ext.ext())
283                    .unwrap(),
284            )
285            .unwrap();
286            let new_code_hash =
287                HexDisplay::from(BlakeTwo256::hash(&new_code).as_fixed_bytes()).to_string();
288            log::info!(
289                target: LOG_TARGET,
290                "New runtime      [Name: {:?}] [Version: {:?}] [Code hash: 0x{}...{}]",
291                new_version.spec_name,
292                new_version.spec_version,
293                &new_code_hash[0..4],
294                &new_code_hash[new_code_hash.len() - 4..],
295            );
296            log::debug!(
297                target: LOG_TARGET,
298                "New runtime code hash: 0x{:?}",
299                new_code_hash
300            );
301
302            if runtime_checks.name_matches && new_version.spec_name != old_version.spec_name {
303                return Err(
304                    "Spec names must match. Use `--disable-spec-name-check` to disable this check."
305                        .into(),
306                );
307            }
308
309            if runtime_checks.version_increases
310                && new_version.spec_version <= old_version.spec_version
311            {
312                return Err(format!("New runtime spec version must be greater than the on-chain runtime spec version: {} <= {}. Use `--disable-spec-version-check` to disable this check.", new_version.spec_version, old_version.spec_version).into());
313            }
314        }
315
316        if runtime_checks.try_runtime_feature_enabled
317            && !ensure_try_runtime::<Block, HostFns>(executor, &mut ext)
318        {
319            return Err("Given runtime is not compiled with the try-runtime feature.".into());
320        }
321
322        Ok(ext)
323    }
324}
325
326/// Build wasm executor by default config.
327pub(crate) fn build_executor<H: HostFunctions>(shared: &SharedParams) -> WasmExecutor<H> {
328    let heap_pages =
329        shared
330            .heap_pages
331            .map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |p| HeapAllocStrategy::Static {
332                extra_pages: p as _,
333            });
334
335    WasmExecutor::builder()
336        .with_execution_method(execution_method_from_cli(
337            shared.wasm_method,
338            shared.wasmtime_instantiation_strategy,
339        ))
340        .with_onchain_heap_alloc_strategy(heap_pages)
341        .with_offchain_heap_alloc_strategy(heap_pages)
342        // There is not that much we can do if someone is using unknown host functions.
343        // They would need to fork the `cli` to add their custom host functions.
344        .with_allow_missing_host_functions(true)
345        .build()
346}
347
348/// Ensure that the given `ext` is compiled with `try-runtime`
349fn ensure_try_runtime<Block: BlockT, HostFns: HostFunctions>(
350    executor: &WasmExecutor<HostFns>,
351    ext: &mut TestExternalities<HashingFor<Block>>,
352) -> bool {
353    use sp_api::RuntimeApiInfo;
354    let final_code = ext
355        .execute_with(|| sp_io::storage::get(well_known_keys::CODE))
356        .expect("':CODE:' is always downloaded in try-runtime-cli; qed");
357    let final_version = <RuntimeVersion as Decode>::decode(
358        &mut &*executor
359            .read_runtime_version(&final_code, &mut ext.ext())
360            .unwrap(),
361    )
362    .unwrap();
363    final_version
364        .api_version(&<dyn frame_try_runtime::TryRuntime<Block>>::ID)
365        .is_some()
366}
367
368/// Execute the given `method` and `data` on top of `ext`, returning the results (encoded) and the
369/// state `changes`.
370pub(crate) fn state_machine_call<Block: BlockT, HostFns: HostFunctions>(
371    ext: &TestExternalities<HashingFor<Block>>,
372    executor: &WasmExecutor<HostFns>,
373    method: &'static str,
374    data: &[u8],
375    mut extensions: Extensions,
376) -> sc_cli::Result<(OverlayedChanges<HashingFor<Block>>, Vec<u8>)> {
377    let mut changes = Default::default();
378    let encoded_result = StateMachine::new(
379        &ext.backend,
380        &mut changes,
381        executor,
382        method,
383        data,
384        &mut extensions,
385        &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend).runtime_code()?,
386        CallContext::Offchain,
387    )
388    .execute()
389    .map_err(|e| format!("failed to execute '{}': {}", method, e))
390    .map_err::<sc_cli::Error, _>(Into::into)?;
391
392    Ok((changes, encoded_result))
393}
394
395/// Same as [`state_machine_call`], but it also computes and returns the storage proof and ref time
396/// information.
397///
398/// Make sure [`LOG_TARGET`] is enabled in logging.
399pub(crate) fn state_machine_call_with_proof<Block: BlockT, HostFns: HostFunctions>(
400    ext: &TestExternalities<HashingFor<Block>>,
401    storage_overlay: &mut OverlayedChanges<HashingFor<Block>>,
402    executor: &WasmExecutor<HostFns>,
403    method: &'static str,
404    data: &[u8],
405    mut extensions: Extensions,
406    maybe_export_proof: Option<PathBuf>,
407) -> sc_cli::Result<(StorageProof, Vec<u8>)> {
408    let runtime_code_backend = sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend);
409    let proving_backend = TrieBackendBuilder::wrap(&ext.backend)
410        .with_recorder(Default::default())
411        .build();
412    let runtime_code = runtime_code_backend.runtime_code()?;
413
414    let encoded_result = StateMachine::new(
415        &proving_backend,
416        storage_overlay,
417        executor,
418        method,
419        data,
420        &mut extensions,
421        &runtime_code,
422        CallContext::Offchain,
423    )
424    .execute()
425    .map_err(|e| format!("failed to execute {}: {}", method, e))
426    .map_err::<sc_cli::Error, _>(Into::into)?;
427
428    let proof = proving_backend
429        .extract_proof()
430        .expect("A recorder was set and thus, a storage proof can be extracted; qed");
431
432    if let Some(path) = maybe_export_proof {
433        let mut file = std::fs::File::create(&path).map_err(|e| {
434            log::error!(
435                target: LOG_TARGET,
436                "Failed to create file {}: {:?}",
437                path.to_string_lossy(),
438                e
439            );
440            e
441        })?;
442
443        log::info!(target: LOG_TARGET, "Writing storage proof to {}", path.to_string_lossy());
444
445        use std::io::Write as _;
446        file.write_all(storage_proof_to_raw_json(&proof).as_bytes())
447            .map_err(|e| {
448                log::error!(
449                    target: LOG_TARGET,
450                    "Failed to write storage proof to {}: {:?}",
451                    path.to_string_lossy(),
452                    e
453                );
454                e
455            })?;
456    }
457
458    Ok((proof, encoded_result))
459}
460
461/// Converts a [`sp_state_machine::StorageProof`] into a JSON string.
462fn storage_proof_to_raw_json(storage_proof: &sp_state_machine::StorageProof) -> String {
463    serde_json::Value::Object(
464        storage_proof
465            .to_memory_db::<sp_runtime::traits::BlakeTwo256>()
466            .drain()
467            .iter()
468            .map(|(key, (value, _n))| {
469                (
470                    format!("0x{}", hex::encode(key.as_bytes())),
471                    serde_json::Value::String(format!("0x{}", hex::encode(value))),
472                )
473            })
474            .collect(),
475    )
476    .to_string()
477}
try_runtime_core/common/state.rs

try_runtime_core/common/
state.rs
