parity_db/
options.rs

1// Copyright 2021-2022 Parity Technologies (UK) Ltd.
2// This file is dual-licensed as Apache-2.0 or MIT.
3
4use crate::{
5	column::{ColId, Salt},
6	compress::CompressionType,
7	error::{try_io, Error, Result},
8};
9use rand::Rng;
10use std::{collections::HashMap, path::Path};
11
12pub const CURRENT_VERSION: u32 = 8;
13// TODO on last supported 5, remove MULTIHEAD_V4 and MULTIPART_V4
14// TODO on last supported 8, remove XOR with salt in column::hash
15const LAST_SUPPORTED_VERSION: u32 = 4;
16
17pub const DEFAULT_COMPRESSION_THRESHOLD: u32 = 4096;
18
19/// Database configuration.
20#[derive(Clone, Debug)]
21pub struct Options {
22	/// Database path.
23	pub path: std::path::PathBuf,
24	/// Column settings
25	pub columns: Vec<ColumnOptions>,
26	/// fsync WAL to disk before committing any changes. Provides extra consistency
27	/// guarantees. On by default.
28	pub sync_wal: bool,
29	/// fsync/msync data to disk before removing logs. Provides crash resistance guarantee.
30	/// On by default.
31	pub sync_data: bool,
32	/// Collect database statistics. May have effect on performance.
33	pub stats: bool,
34	/// Override salt value. If `None` is specified salt is loaded from metadata
35	/// or randomly generated when creating a new database.
36	pub salt: Option<Salt>,
37	/// Minimal value size threshold to attempt compressing a value per column.
38	///
39	/// Optional. A sensible default is used if nothing is set for a given column.
40	pub compression_threshold: HashMap<ColId, u32>,
41	#[cfg(any(test, feature = "instrumentation"))]
42	/// Always starts background threads.
43	pub with_background_thread: bool,
44	#[cfg(any(test, feature = "instrumentation"))]
45	/// Always flushes data from the log to the on-disk data structures.
46	pub always_flush: bool,
47}
48
49/// Database column configuration.
50#[derive(Clone, Debug, PartialEq, Eq)]
51pub struct ColumnOptions {
52	/// Indicates that the column value is the preimage of the key.
53	/// This implies that a given value always has the same key.
54	/// Enables some optimizations.
55	pub preimage: bool,
56	/// Indicates that the keys are at least 32 bytes and
57	/// the first 32 bytes have uniform distribution.
58	/// Allows for skipping additional key hashing.
59	pub uniform: bool,
60	/// Use reference counting for values. Reference operations are allowed for a column. The value
61	/// is deleted when the counter reaches zero.
62	pub ref_counted: bool,
63	/// Compression to use for this column.
64	pub compression: CompressionType,
65	/// Column is configured to use Btree storage. Btree columns allow for ordered key iteration
66	/// and key retrieval, but are significantly less performant and require more disk space.
67	pub btree_index: bool,
68}
69
70/// Database metadata.
71#[derive(Clone, Debug)]
72pub struct Metadata {
73	/// Salt value.
74	pub salt: Salt,
75	/// Database version.
76	pub version: u32,
77	/// Column metadata.
78	pub columns: Vec<ColumnOptions>,
79}
80
81impl ColumnOptions {
82	fn as_string(&self) -> String {
83		format!(
84			"preimage: {}, uniform: {}, refc: {}, compression: {}, ordered: {}",
85			self.preimage, self.uniform, self.ref_counted, self.compression as u8, self.btree_index,
86		)
87	}
88
89	pub fn is_valid(&self) -> bool {
90		if self.ref_counted && !self.preimage {
91			log::error!(target: "parity-db", "Using `ref_counted` option without `preimage` enabled is not supported");
92			return false
93		}
94		true
95	}
96
97	fn from_string(s: &str) -> Option<Self> {
98		let mut split = s.split("sizes: ");
99		let vals = split.next()?;
100
101		let vals: HashMap<&str, &str> = vals
102			.split(", ")
103			.filter_map(|s| {
104				let mut pair = s.split(": ");
105				Some((pair.next()?, pair.next()?))
106			})
107			.collect();
108
109		let preimage = vals.get("preimage")?.parse().ok()?;
110		let uniform = vals.get("uniform")?.parse().ok()?;
111		let ref_counted = vals.get("refc")?.parse().ok()?;
112		let compression: u8 = vals.get("compression").and_then(|c| c.parse().ok()).unwrap_or(0);
113		let btree_index = vals.get("ordered").and_then(|c| c.parse().ok()).unwrap_or(false);
114
115		Some(ColumnOptions {
116			preimage,
117			uniform,
118			ref_counted,
119			compression: compression.into(),
120			btree_index,
121		})
122	}
123}
124
125impl Default for ColumnOptions {
126	fn default() -> ColumnOptions {
127		ColumnOptions {
128			preimage: false,
129			uniform: false,
130			ref_counted: false,
131			compression: CompressionType::NoCompression,
132			btree_index: false,
133		}
134	}
135}
136
137impl Options {
138	pub fn with_columns(path: &Path, num_columns: u8) -> Options {
139		Options {
140			path: path.into(),
141			sync_wal: true,
142			sync_data: true,
143			stats: true,
144			salt: None,
145			columns: (0..num_columns).map(|_| Default::default()).collect(),
146			compression_threshold: HashMap::new(),
147			#[cfg(any(test, feature = "instrumentation"))]
148			with_background_thread: true,
149			#[cfg(any(test, feature = "instrumentation"))]
150			always_flush: false,
151		}
152	}
153
154	// TODO on next major version remove in favor of write_metadata_with_version
155	pub fn write_metadata(&self, path: &Path, salt: &Salt) -> Result<()> {
156		self.write_metadata_with_version(path, salt, None)
157	}
158
159	// TODO on next major version remove in favor of write_metadata_with_version
160	pub fn write_metadata_file(&self, path: &Path, salt: &Salt) -> Result<()> {
161		self.write_metadata_file_with_version(path, salt, None)
162	}
163
164	pub fn write_metadata_with_version(
165		&self,
166		path: &Path,
167		salt: &Salt,
168		version: Option<u32>,
169	) -> Result<()> {
170		let mut path = path.to_path_buf();
171		path.push("metadata");
172		self.write_metadata_file_with_version(&path, salt, version)
173	}
174
175	pub fn write_metadata_file_with_version(
176		&self,
177		path: &Path,
178		salt: &Salt,
179		version: Option<u32>,
180	) -> Result<()> {
181		let mut metadata = vec![
182			format!("version={}", version.unwrap_or(CURRENT_VERSION)),
183			format!("salt={}", hex::encode(salt)),
184		];
185		for i in 0..self.columns.len() {
186			metadata.push(format!("col{}={}", i, self.columns[i].as_string()));
187		}
188		try_io!(std::fs::write(path, metadata.join("\n")));
189		Ok(())
190	}
191
192	pub fn load_and_validate_metadata(&self, create: bool) -> Result<Metadata> {
193		let meta = Self::load_metadata(&self.path)?;
194
195		if let Some(meta) = meta {
196			if meta.columns.len() != self.columns.len() {
197				return Err(Error::InvalidConfiguration(format!(
198					"Column config mismatch. Expected {} columns, got {}",
199					self.columns.len(),
200					meta.columns.len()
201				)))
202			}
203
204			for c in 0..meta.columns.len() {
205				if meta.columns[c] != self.columns[c] {
206					return Err(Error::IncompatibleColumnConfig {
207						id: c as ColId,
208						reason: format!(
209							"Column config mismatch. Expected \"{}\", got \"{}\"",
210							self.columns[c].as_string(),
211							meta.columns[c].as_string(),
212						),
213					})
214				}
215			}
216			Ok(meta)
217		} else if create {
218			let s: Salt = self.salt.unwrap_or_else(|| rand::thread_rng().gen());
219			self.write_metadata(&self.path, &s)?;
220			Ok(Metadata { version: CURRENT_VERSION, columns: self.columns.clone(), salt: s })
221		} else {
222			Err(Error::DatabaseNotFound)
223		}
224	}
225
226	pub fn load_metadata(path: &Path) -> Result<Option<Metadata>> {
227		let mut path = path.to_path_buf();
228		path.push("metadata");
229		Self::load_metadata_file(&path)
230	}
231
232	pub fn load_metadata_file(path: &Path) -> Result<Option<Metadata>> {
233		use std::{io::BufRead, str::FromStr};
234
235		if !path.exists() {
236			return Ok(None)
237		}
238		let file = std::io::BufReader::new(try_io!(std::fs::File::open(path)));
239		let mut salt = None;
240		let mut columns = Vec::new();
241		let mut version = 0;
242		for l in file.lines() {
243			let l = try_io!(l);
244			let mut vals = l.split('=');
245			let k = vals.next().ok_or_else(|| Error::Corruption("Bad metadata".into()))?;
246			let v = vals.next().ok_or_else(|| Error::Corruption("Bad metadata".into()))?;
247			if k == "version" {
248				version =
249					u32::from_str(v).map_err(|_| Error::Corruption("Bad version string".into()))?;
250			} else if k == "salt" {
251				let salt_slice =
252					hex::decode(v).map_err(|_| Error::Corruption("Bad salt string".into()))?;
253				let mut s = Salt::default();
254				s.copy_from_slice(&salt_slice);
255				salt = Some(s);
256			} else if k.starts_with("col") {
257				let col = ColumnOptions::from_string(v)
258					.ok_or_else(|| Error::Corruption("Bad column metadata".into()))?;
259				columns.push(col);
260			}
261		}
262		if version < LAST_SUPPORTED_VERSION {
263			return Err(Error::InvalidConfiguration(format!(
264				"Unsupported database version {version}. Expected {CURRENT_VERSION}"
265			)))
266		}
267		let salt = salt.ok_or_else(|| Error::InvalidConfiguration("Missing salt value".into()))?;
268		Ok(Some(Metadata { version, columns, salt }))
269	}
270
271	pub fn is_valid(&self) -> bool {
272		for option in self.columns.iter() {
273			if !option.is_valid() {
274				return false
275			}
276		}
277		true
278	}
279}
280
281impl Metadata {
282	pub fn columns_to_migrate(&self) -> std::collections::BTreeSet<u8> {
283		std::collections::BTreeSet::new()
284	}
285}
parity_db/options.rs

parity_db/
options.rs
