try_runtime_core/common/
state.rs

1// This file is part of try-runtime-cli.
2
3// Copyright (C) Parity Technologies (UK) Ltd.
4// SPDX-License-Identifier: Apache-2.0
5
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9//
10// 	http://www.apache.org/licenses/LICENSE-2.0
11//
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17
18use std::{fmt::Debug, path::PathBuf, str::FromStr};
19
20use frame_remote_externalities::{
21    Builder, Mode, OfflineConfig, OnlineConfig, RemoteExternalities, SnapshotConfig,
22};
23use parity_scale_codec::Decode;
24use sc_cli::{execution_method_from_cli, RuntimeVersion};
25use sc_executor::{
26    sp_wasm_interface::HostFunctions, HeapAllocStrategy, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY,
27};
28use sp_api::{CallContext, StorageProof};
29use sp_core::{
30    hexdisplay::HexDisplay, storage::well_known_keys, traits::ReadRuntimeVersion, Hasher,
31};
32use sp_crypto_hashing::twox_128;
33use sp_externalities::Extensions;
34use sp_runtime::{
35    traits::{BlakeTwo256, Block as BlockT, HashingFor, Header as HeaderT},
36    DeserializeOwned,
37};
38use sp_state_machine::{OverlayedChanges, StateMachine, TestExternalities, TrieBackendBuilder};
39use substrate_rpc_client::{ws_client, ChainApi};
40
41use crate::{
42    common::{
43        parse,
44        shared_parameters::{Runtime, SharedParams},
45    },
46    hash_of, rpc_err_handler, LOG_TARGET,
47};
48
49/// A `Live` variant for [`State`]
50#[derive(Debug, Clone, clap::Args)]
51pub struct LiveState {
52    /// The url(s) to connect to. Can be provided multiple times for parallel state download.
53    #[arg(
54		short,
55		long,
56		value_parser = parse::url,
57		num_args = 1..,
58		required = true,
59	)]
60    pub uri: Vec<String>,
61
62    /// The block hash at which to fetch the state.
63    ///
64    /// If non provided, then the latest finalized head is used.
65    #[arg(
66		short,
67		long,
68		value_parser = parse::hash,
69	)]
70    pub at: Option<String>,
71
72    /// A pallet to scrape. Can be provided multiple times. If empty, entire chain state will
73    /// be scraped.
74    ///
75    /// This is equivalent to passing `xx_hash_64(pallet)` to `--hashed_prefixes`.
76    #[arg(short, long, num_args = 1..)]
77    pub pallet: Vec<String>,
78
79    /// Storage entry key prefixes to scrape and inject into the test externalities. Pass as 0x
80    /// prefixed hex strings. By default, all keys are scraped and included.
81    #[arg(long = "prefix", value_parser = parse::hash, num_args = 1..)]
82    pub hashed_prefixes: Vec<String>,
83
84    /// Fetch the child-keys as well.
85    ///
86    /// Default is `false`, if specific `--pallets` are specified, `true` otherwise. In other
87    /// words, if you scrape the whole state the child tree data is included out of the box.
88    /// Otherwise, it must be enabled explicitly using this flag.
89    #[arg(long)]
90    pub child_tree: bool,
91}
92
93impl LiveState {
94    /// Return the `at` block hash as a `Hash`, if it exists.
95    pub fn at<Block: BlockT>(&self) -> sc_cli::Result<Option<<Block>::Hash>>
96    where
97        <Block::Hash as FromStr>::Err: Debug,
98    {
99        self.at
100            .clone()
101            .map(|s| hash_of::<Block>(s.as_str()))
102            .transpose()
103    }
104
105    /// Converts this `LiveState` into a `LiveState` for the previous block.
106    ///
107    /// Useful for opertations like when you want to execute a block, but also need the state of the
108    /// block *before* it.
109    pub async fn to_prev_block_live_state<Block: BlockT>(self) -> sc_cli::Result<LiveState>
110    where
111        <Block::Hash as FromStr>::Err: Debug,
112    {
113        // We want to execute the block `at`, therefore need the state of the block *before* it.
114        let at = self.at::<Block>()?;
115
116        // Get the block number requested by the user, or the current block number if they
117        // didn't specify one.
118        let rpc = ws_client(&self.uri[0]).await?;
119        let previous_hash = ChainApi::<(), Block::Hash, Block::Header, ()>::header(&rpc, at)
120            .await
121            .map_err(rpc_err_handler)
122            .and_then(|maybe_header| {
123                maybe_header
124                    .ok_or("header_not_found")
125                    .map(|h| *h.parent_hash())
126            })?;
127
128        Ok(LiveState {
129            at: Some(hex::encode(previous_hash)),
130            ..self
131        })
132    }
133}
134
135/// The source of runtime *state* to use.
136#[derive(Debug, Clone, clap::Subcommand)]
137pub enum State {
138    /// Use a state snapshot as the source of runtime state.
139    Snap {
140        #[clap(short = 'p', long = "path", alias = "snapshot-path")]
141        path: Option<PathBuf>,
142    },
143
144    /// Use a live chain as the source of runtime state.
145    Live(LiveState),
146}
147
148/// Checks to perform on the given runtime, compared to the existing runtime.
149#[derive(Debug, Clone, Copy)]
150pub struct RuntimeChecks {
151    /// Enforce the `spec_name`s match
152    pub name_matches: bool,
153    /// Enforce the `spec_version` of the given is greater or equal to the existing
154    /// runtime.
155    pub version_increases: bool,
156    /// Enforce that the given runtime is compiled with the try-runtime feature.
157    pub try_runtime_feature_enabled: bool,
158}
159
160impl State {
161    /// Create the [`RemoteExternalities`].
162    ///
163    /// This will override the code as it sees fit based on [`Runtime`]. It will also check the
164    /// spec-version and name.
165    pub async fn to_ext<Block: BlockT + DeserializeOwned, HostFns: HostFunctions>(
166        &self,
167        shared: &SharedParams,
168        executor: &WasmExecutor<HostFns>,
169        state_snapshot: Option<SnapshotConfig>,
170        runtime_checks: RuntimeChecks,
171    ) -> sc_cli::Result<RemoteExternalities<Block>>
172    where
173        Block::Header: DeserializeOwned,
174        <Block::Hash as FromStr>::Err: Debug,
175    {
176        let builder = match self {
177            State::Snap { path } => {
178                let path = path
179                    .as_ref()
180                    .ok_or_else(|| "no snapshot path provided".to_string())?;
181
182                Builder::<Block>::new().mode(Mode::Offline(OfflineConfig {
183                    state_snapshot: SnapshotConfig::new(path),
184                }))
185            }
186            State::Live(LiveState {
187                pallet,
188                uri,
189                at,
190                child_tree,
191                hashed_prefixes,
192            }) => {
193                let at = match at {
194                    Some(at_str) => Some(hash_of::<Block>(at_str)?),
195                    None => None,
196                };
197                let hashed_prefixes = hashed_prefixes
198                    .iter()
199                    .map(|p_str| {
200                        hex::decode(p_str).map_err(|e| {
201                            format!(
202                                "Error decoding `hashed_prefixes` hex string entry '{:?}' to bytes: {:?}",
203                                p_str, e
204                            )
205                        })
206                    })
207                    .collect::<Result<Vec<_>, _>>()?;
208                Builder::<Block>::new().mode(Mode::Online(OnlineConfig {
209                    at,
210                    transport_uris: uri.clone(),
211                    state_snapshot,
212                    pallets: pallet.clone(),
213                    child_trie: *child_tree,
214                    hashed_keys: vec![
215                        // we always download the code, but we almost always won't use it, based on
216                        // `Runtime`.
217                        well_known_keys::CODE.to_vec(),
218                        // we will always download this key, since it helps detect if we should do
219                        // runtime migration or not.
220                        [twox_128(b"System"), twox_128(b"LastRuntimeUpgrade")].concat(),
221                        [twox_128(b"System"), twox_128(b"Number")].concat(),
222                    ],
223                    hashed_prefixes,
224                }))
225            }
226        };
227
228        // possibly overwrite the state version, should hardly be needed.
229        let builder = if let Some(state_version) = shared.overwrite_state_version {
230            log::warn!(
231                target: LOG_TARGET,
232                "overwriting state version to {:?}, you better know what you are doing.",
233                state_version
234            );
235            builder.overwrite_state_version(state_version)
236        } else {
237            builder
238        };
239
240        // then, we prepare to replace the code based on what the CLI wishes.
241        let maybe_code_to_overwrite = match shared.runtime {
242            Runtime::Path(ref path) => Some(std::fs::read(path).map_err(|e| {
243                format!("error while reading runtime file from {:?}: {:?}", path, e)
244            })?),
245            Runtime::Existing => None,
246        };
247
248        // build the main ext.
249        let mut ext = builder.build().await?;
250
251        // actually replace the code if needed.
252        if let Some(new_code) = maybe_code_to_overwrite {
253            let original_code = ext
254                .execute_with(|| sp_io::storage::get(well_known_keys::CODE))
255                .expect("':CODE:' is always downloaded in try-runtime-cli; qed");
256
257            // NOTE: see the impl notes of `read_runtime_version`, the ext is almost not used here,
258            // only as a backup.
259            ext.insert(well_known_keys::CODE.to_vec(), new_code.clone());
260            let old_version = <RuntimeVersion as Decode>::decode(
261                &mut &*executor
262                    .read_runtime_version(&original_code, &mut ext.ext())
263                    .unwrap(),
264            )
265            .unwrap();
266            let old_code_hash =
267                HexDisplay::from(BlakeTwo256::hash(&original_code).as_fixed_bytes()).to_string();
268            log::info!(
269                target: LOG_TARGET,
270                "Original runtime [Name: {:?}] [Version: {:?}] [Code hash: 0x{}...{}]",
271                old_version.spec_name,
272                old_version.spec_version,
273                &old_code_hash[0..4],
274                &old_code_hash[old_code_hash.len() - 4..],
275            );
276            log::debug!(
277                target: LOG_TARGET,
278                "Original runtime full code hash: 0x{:?}",
279                old_code_hash,
280            );
281            let new_version = <RuntimeVersion as Decode>::decode(
282                &mut &*executor
283                    .read_runtime_version(&new_code, &mut ext.ext())
284                    .unwrap(),
285            )
286            .unwrap();
287            let new_code_hash =
288                HexDisplay::from(BlakeTwo256::hash(&new_code).as_fixed_bytes()).to_string();
289            log::info!(
290                target: LOG_TARGET,
291                "New runtime      [Name: {:?}] [Version: {:?}] [Code hash: 0x{}...{}]",
292                new_version.spec_name,
293                new_version.spec_version,
294                &new_code_hash[0..4],
295                &new_code_hash[new_code_hash.len() - 4..],
296            );
297            log::debug!(
298                target: LOG_TARGET,
299                "New runtime code hash: 0x{:?}",
300                new_code_hash
301            );
302
303            if runtime_checks.name_matches && new_version.spec_name != old_version.spec_name {
304                return Err(
305                    "Spec names must match. Use `--disable-spec-name-check` to disable this check."
306                        .into(),
307                );
308            }
309
310            if runtime_checks.version_increases
311                && new_version.spec_version <= old_version.spec_version
312            {
313                return Err(format!("New runtime spec version must be greater than the on-chain runtime spec version: {} <= {}. Use `--disable-spec-version-check` to disable this check.", new_version.spec_version, old_version.spec_version).into());
314            }
315        }
316
317        if runtime_checks.try_runtime_feature_enabled
318            && !ensure_try_runtime::<Block, HostFns>(executor, &mut ext)
319        {
320            return Err("Given runtime is not compiled with the try-runtime feature.".into());
321        }
322
323        Ok(ext)
324    }
325}
326
327/// Build wasm executor by default config.
328pub(crate) fn build_executor<H: HostFunctions>(shared: &SharedParams) -> WasmExecutor<H> {
329    let heap_pages =
330        shared
331            .heap_pages
332            .map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |p| HeapAllocStrategy::Static {
333                extra_pages: p as _,
334            });
335
336    WasmExecutor::builder()
337        .with_execution_method(execution_method_from_cli(
338            shared.wasm_method,
339            shared.wasmtime_instantiation_strategy,
340        ))
341        .with_onchain_heap_alloc_strategy(heap_pages)
342        .with_offchain_heap_alloc_strategy(heap_pages)
343        // There is not that much we can do if someone is using unknown host functions.
344        // They would need to fork the `cli` to add their custom host functions.
345        .with_allow_missing_host_functions(true)
346        .build()
347}
348
349/// Ensure that the given `ext` is compiled with `try-runtime`
350fn ensure_try_runtime<Block: BlockT, HostFns: HostFunctions>(
351    executor: &WasmExecutor<HostFns>,
352    ext: &mut TestExternalities<HashingFor<Block>>,
353) -> bool {
354    use sp_api::RuntimeApiInfo;
355    let final_code = ext
356        .execute_with(|| sp_io::storage::get(well_known_keys::CODE))
357        .expect("':CODE:' is always downloaded in try-runtime-cli; qed");
358    let final_version = <RuntimeVersion as Decode>::decode(
359        &mut &*executor
360            .read_runtime_version(&final_code, &mut ext.ext())
361            .unwrap(),
362    )
363    .unwrap();
364    final_version
365        .api_version(&<dyn frame_try_runtime::TryRuntime<Block>>::ID)
366        .is_some()
367}
368
369/// Execute the given `method` and `data` on top of `ext`, returning the results (encoded) and the
370/// state `changes`.
371pub(crate) fn state_machine_call<Block: BlockT, HostFns: HostFunctions>(
372    ext: &TestExternalities<HashingFor<Block>>,
373    executor: &WasmExecutor<HostFns>,
374    method: &'static str,
375    data: &[u8],
376    mut extensions: Extensions,
377) -> sc_cli::Result<(OverlayedChanges<HashingFor<Block>>, Vec<u8>)> {
378    let mut changes = Default::default();
379    let encoded_result = StateMachine::new(
380        &ext.backend,
381        &mut changes,
382        executor,
383        method,
384        data,
385        &mut extensions,
386        &sp_state_machine::backend::BackendRuntimeCode::new(
387            &ext.backend,
388            sp_state_machine::backend::TryPendingCode::No,
389        )
390        .runtime_code()?,
391        CallContext::Offchain,
392    )
393    .execute()
394    .map_err(|e| format!("failed to execute '{}': {}", method, e))
395    .map_err::<sc_cli::Error, _>(Into::into)?;
396
397    Ok((changes, encoded_result))
398}
399
400/// Same as [`state_machine_call`], but it also computes and returns the storage proof and ref time
401/// information.
402///
403/// Make sure [`LOG_TARGET`] is enabled in logging.
404pub(crate) fn state_machine_call_with_proof<Block: BlockT, HostFns: HostFunctions>(
405    ext: &TestExternalities<HashingFor<Block>>,
406    storage_overlay: &mut OverlayedChanges<HashingFor<Block>>,
407    executor: &WasmExecutor<HostFns>,
408    method: &'static str,
409    data: &[u8],
410    mut extensions: Extensions,
411    maybe_export_proof: Option<PathBuf>,
412) -> sc_cli::Result<(StorageProof, Vec<u8>)> {
413    let runtime_code_backend = sp_state_machine::backend::BackendRuntimeCode::new(
414        &ext.backend,
415        sp_state_machine::backend::TryPendingCode::No,
416    );
417    let proving_backend = TrieBackendBuilder::wrap(&ext.backend)
418        .with_recorder(Default::default())
419        .build();
420    let runtime_code = runtime_code_backend.runtime_code()?;
421
422    let encoded_result = StateMachine::new(
423        &proving_backend,
424        storage_overlay,
425        executor,
426        method,
427        data,
428        &mut extensions,
429        &runtime_code,
430        CallContext::Offchain,
431    )
432    .execute()
433    .map_err(|e| format!("failed to execute {}: {}", method, e))
434    .map_err::<sc_cli::Error, _>(Into::into)?;
435
436    let proof = proving_backend
437        .extract_proof()
438        .expect("A recorder was set and thus, a storage proof can be extracted; qed");
439
440    if let Some(path) = maybe_export_proof {
441        let mut file = std::fs::File::create(&path).map_err(|e| {
442            log::error!(
443                target: LOG_TARGET,
444                "Failed to create file {}: {:?}",
445                path.to_string_lossy(),
446                e
447            );
448            e
449        })?;
450
451        log::info!(target: LOG_TARGET, "Writing storage proof to {}", path.to_string_lossy());
452
453        use std::io::Write as _;
454        file.write_all(storage_proof_to_raw_json(&proof).as_bytes())
455            .map_err(|e| {
456                log::error!(
457                    target: LOG_TARGET,
458                    "Failed to write storage proof to {}: {:?}",
459                    path.to_string_lossy(),
460                    e
461                );
462                e
463            })?;
464    }
465
466    Ok((proof, encoded_result))
467}
468
469/// Converts a [`sp_state_machine::StorageProof`] into a JSON string.
470fn storage_proof_to_raw_json(storage_proof: &sp_state_machine::StorageProof) -> String {
471    serde_json::Value::Object(
472        storage_proof
473            .to_memory_db::<sp_runtime::traits::BlakeTwo256>()
474            .drain()
475            .iter()
476            .map(|(key, (value, _n))| {
477                (
478                    format!("0x{}", hex::encode(key.as_bytes())),
479                    serde_json::Value::String(format!("0x{}", hex::encode(value))),
480                )
481            })
482            .collect(),
483    )
484    .to_string()
485}
try_runtime_core/common/state.rs

try_runtime_core/common/
state.rs
