awesome-android/avbroot
3
0
Fork 0
mirror of https://github.com/chenxiaolong/avbroot.git synced 2026-07-03 14:05:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #146 from chenxiaolong/fec

Browse Source 
Add support for AVB FEC
This commit is contained in:
Andrew Gunnerson
2023-09-18 16:37:17 -04:00
committed by GitHub
parent fb7be77480 acc3308d37
commit 6ca4023b0d
13 changed files with 1408 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+90 -8
View File
@@ -122,6 +122,7 @@ dependencies = [
"const-oid",
"ctrlc",
"flate2",
"gf256",
"hex",
"lz4_flex",
"memchr",
@@ -304,7 +305,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
]
[[package]]
@@ -434,6 +435,41 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "darling"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a"
dependencies = [
"darling_core",
"quote",
"syn 1.0.109",
]
[[package]]
name = "der"
version = "0.7.8"
@@ -455,7 +491,7 @@ checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
]
[[package]]
@@ -520,6 +556,12 @@ dependencies = [
"libc",
]
[[package]]
name = "evalexpr"
version = "6.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfc25fd417983cc7f203394ebb89eba18e2df1b0ac1be2673091b5aca52b595f"
[[package]]
name = "fastrand"
version = "2.0.0"
@@ -593,6 +635,29 @@ dependencies = [
"wasi",
]
[[package]]
name = "gf256"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00ad9760c1b289e26006c2ed8cc950807c842e6c51f64532c806bc2ef7a34c79"
dependencies = [
"cfg-if",
"gf256-macros",
]
[[package]]
name = "gf256-macros"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b084923ea97216d84ff299a428609b85e68c130c5320a86ed778c204f9e19cd"
dependencies = [
"darling",
"evalexpr",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "hashbrown"
version = "0.14.0"
@@ -646,6 +711,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "140a09c9305e6d5e557e2ed7cbc68e05765a7d4213975b87cb04920689cc6219"
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
version = "0.4.0"
@@ -900,7 +971,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
]
[[package]]
@@ -986,7 +1057,7 @@ dependencies = [
"phf_shared",
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
]
[[package]]
@@ -1322,7 +1393,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
]
[[package]]
@@ -1412,6 +1483,17 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
[[package]]
name = "syn"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.29"
@@ -1453,7 +1535,7 @@ checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
]
[[package]]
@@ -1608,7 +1690,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
"wasm-bindgen-shared",
]
@@ -1630,7 +1712,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
README.extra.md
+58 -1
View File
@@ -20,7 +20,7 @@ This subcommand shows all of the vbmeta header and footer fields. `vbmeta` parti
avbroot avb verify -i <root vbmeta image> -p <public key>
```
This subcommand verifies the vbmeta header signature and the hashes for all vbmeta descriptors (including hashtree descriptors). If the vbmeta image has a chain descriptor for another partition, that partition image will be verified as well (recursively). All partitions are expected to be in the same directory as the vbmeta image being verified.
This subcommand verifies the vbmeta header signature and the hashes for all vbmeta descriptors (including hash tree descriptors). If the vbmeta image has a chain descriptor for another partition, that partition image will be verified as well (recursively). All partitions are expected to be in the same directory as the vbmeta image being verified.
If `-p` is omitted, the signatures and hashes are checked only for validity, not that they are trusted.
@@ -58,6 +58,63 @@ avbroot boot info -i <input boot image>
All of the `boot` subcommands show the boot image information. This specific subcommand just does it without performing any other operation. To show avbroot's internal representation of the information, pass in `-d`.
## `avbroot fec`
This set of commands is for working with dm-verity FEC (forward error correction) data. The FEC data allows small errors in partition data to be corrected. This increases reliability of the system because when dm-verity encounters data that doesn't match the expected checksum, it will either trigger a kernel panic or reboot the system.
The same raw FEC data can be stored in several ways:
* cryptsetup's `veritysetup` does not use any file format at all. It must be told the FEC location and parameters using the `--fec-*` options.
* AOSP's AVB 2.0 stores the FEC data inside the partition as `[Partition data][Hash tree][FEC data]`. The location and parameters are stored in the vbmeta hash tree descriptors.
* AOSP's `fec` tool stores the FEC data in a standalone file with a header containing the FEC parameters.
The `avbroot fec` commands use AOSP's standalone FEC file format.
The FEC data is not generated from a sequential read of the input file, but rather from an interleaved read. If the input file's offsets are visualized as a 2D table:
```
| 0 1 2 3 ... 4095 |
| 4096 4097 4098 4099 ... 8191 |
| 8192 8193 8194 8195 ... 12287 |
| .... .... .... .... ... ..... |
```
then the file access pattern can be thought of as being column-by-column instead of row-by-row.
Data correction happens at the codeword level. A Reed-Solomon codeword is 255 bytes where some portion is file data and the rest is parity data. AOSP and avbroot both default to 253 bytes of data and 2 bytes of parity information. Each column in the table represents the 253-byte data portion of the codeword. Larger files have more columns.
A contiguous sequence of corrupted data will span multiple columns. Since error correction happens at the column level, this interleaving increases the chances of recovery. For more details about the specifics, see the implementation in [`fec.rs`](./avbroot/src/format/fec.rs).
### Generating FEC data
```bash
avbroot fec generate -i <input data file> -f <output FEC file>
```
The default behavior is to use 2 bytes of parity information per 253 bytes of input data. Within each 253-byte column described above, this is sufficient for correcting a single corrupted byte in the column (`⌊parity / 2⌋` bytes in general).
The number of parity bytes (between 2 and 24, inclusive) can be configured using `--parity`.
### Verifying a file
```bash
avbroot fec verify -i <input data file> -f <input FEC file>
```
This will check if the input file has any corrupted bytes. This command runs significantly faster than `avbroot fec repair` and is useful if only detection of corrupted data is needed.
Note that FEC is **not** a replacement for checksums, like SHA-256. When there are too many errors, there can be false positives where the corrupted data is reported as being valid.
### Repairing a file
```bash
avbroot fec repair -i <input/output data file> -f <input FEC file>
```
This will repair the file in place. As described above, in each column, up to `parity / 2` bytes can be corrected.
Note that FEC is **not** a replacement for checksums, like SHA-256. When there are too many errors, the file can potentially be "successfully repaired" to some incorrect data.
## `avbroot ramdisk`
### Dumping a cpio archive
avbroot/Cargo.toml
+1
View File
@@ -18,6 +18,7 @@ cms = { version = "0.2.2", features = ["std"] }
const-oid = "0.9.5"
ctrlc = "3.4.0"
flate2 = "1.0.27"
gf256 = { version = "0.3.0", features = ["rs"] }
hex = "0.4.3"
lz4_flex = "0.11.1"
memchr = "2.6.0"
avbroot/src/cli/args.rs
+3 -1
View File
@@ -8,7 +8,7 @@ use std::sync::atomic::AtomicBool;
use anyhow::Result;
use clap::{Parser, Subcommand};
use crate::cli::{avb, boot, completion, key, ota, ramdisk};
use crate::cli::{avb, boot, completion, fec, key, ota, ramdisk};
#[allow(clippy::large_enum_variant)]
#[derive(Debug, Subcommand)]
@@ -16,6 +16,7 @@ pub enum Command {
Avb(avb::AvbCli),
Boot(boot::BootCli),
Completion(completion::CompletionCli),
Fec(fec::FecCli),
Key(key::KeyCli),
Ota(ota::OtaCli),
Ramdisk(ramdisk::RamdiskCli),
@@ -41,6 +42,7 @@ pub fn main(cancel_signal: &AtomicBool) -> Result<()> {
Command::Avb(c) => avb::avb_main(&c, cancel_signal),
Command::Boot(c) => boot::boot_main(&c),
Command::Completion(c) => completion::completion_main(&c),
Command::Fec(c) => fec::fec_main(&c, cancel_signal),
Command::Key(c) => key::key_main(&c),
Command::Ota(c) => ota::ota_main(&c, cancel_signal),
Command::Ramdisk(c) => ramdisk::ramdisk_main(&c),
avbroot/src/cli/avb.rs
+10 -5
View File
@@ -34,7 +34,7 @@ fn ensure_name_is_safe(name: &str) -> Result<()> {
/// Recursively verify an image's vbmeta header and all of the chained images.
/// `seen` is used to prevent cycles. `descriptors` will contain all of the hash
/// and hashtree descriptors that need to be verified.
/// and hash tree descriptors that need to be verified.
pub fn verify_headers(
directory: &Path,
name: &str,
@@ -81,7 +81,7 @@ pub fn verify_headers(
};
match descriptor {
avb::Descriptor::Hashtree(_) | avb::Descriptor::Hash(_) => {
avb::Descriptor::HashTree(_) | avb::Descriptor::Hash(_) => {
if let Some(prev) = descriptors.get(target_name) {
if prev != descriptor {
bail!("{name} descriptor does not match previous encounter");
@@ -128,13 +128,15 @@ pub fn verify_descriptors(
};
match descriptor {
Descriptor::Hashtree(d) => {
status!("Verifying hashtree descriptor for: {name}");
Descriptor::HashTree(d) => {
status!("Verifying hash tree descriptor for: {name}");
d.verify(
|| Ok(Box::new(BufReader::new(reader.clone()))),
cancel_signal,
)
.with_context(|| format!("Failed to verify hashtree descriptor for: {name}"))?;
.with_context(|| {
format!("Failed to verify hash tree descriptor for: {name}")
})?;
}
Descriptor::Hash(d) => {
status!("Verifying hash descriptor for: {name}");
@@ -209,6 +211,9 @@ struct DumpCli {
}
/// Verify vbmeta signatures.
///
/// For hash tree descriptors, the FEC (forward error correction) data will also
/// be verified.
#[derive(Debug, Parser)]
struct VerifyCli {
/// Path to input image.
avbroot/src/cli/fec.rs
+153
View File
@@ -0,0 +1,153 @@
/*
* SPDX-FileCopyrightText: 2023 Andrew Gunnerson
* SPDX-License-Identifier: GPL-3.0-only
*/
use std::{
fs::{File, OpenOptions},
io::{BufReader, BufWriter, Write},
path::{Path, PathBuf},
sync::atomic::AtomicBool,
};
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use crate::{
format::fec::FecImage,
stream::{FromReader, PSeekFile, ToWriter},
};
fn open_input(path: &Path, rw: bool) -> Result<PSeekFile> {
OpenOptions::new()
.read(true)
.write(rw)
.open(path)
.map(PSeekFile::new)
.with_context(|| format!("Failed to open file: {path:?}"))
}
fn read_fec(path: &Path) -> Result<FecImage> {
let reader = File::open(path)
.map(BufReader::new)
.with_context(|| format!("Failed to open for reading: {path:?}"))?;
let fec = FecImage::from_reader(reader)
.with_context(|| format!("Failed to read FEC data: {path:?}"))?;
Ok(fec)
}
fn write_fec(path: &Path, fec: &FecImage) -> Result<()> {
let mut writer = File::create(path)
.map(BufWriter::new)
.with_context(|| format!("Failed to open for writing: {path:?}"))?;
fec.to_writer(&mut writer)
.with_context(|| format!("Failed to write FEC data: {path:?}"))?;
writer.flush()?;
Ok(())
}
fn generate_subcommand(cli: &GenerateCli, cancel_signal: &AtomicBool) -> Result<()> {
let input = open_input(&cli.input, false)?;
let fec = FecImage::generate(|| Ok(Box::new(input.clone())), cli.parity, cancel_signal)
.context("Failed to generate FEC data")?;
write_fec(&cli.fec, &fec)?;
Ok(())
}
fn verify_subcommand(cli: &VerifyCli, cancel_signal: &AtomicBool) -> Result<()> {
let input = open_input(&cli.input, false)?;
let fec = read_fec(&cli.fec)?;
fec.verify(|| Ok(Box::new(input.clone())), cancel_signal)
.context("Failed to verify data")?;
Ok(())
}
fn repair_subcommand(cli: &RepairCli, cancel_signal: &AtomicBool) -> Result<()> {
let input = open_input(&cli.input, true)?;
let fec = read_fec(&cli.fec)?;
// The separate buffered readers and writers are safe because the function
// guarantees that every thread touches disjoint offsets and every offset is
// read and written at most once.
fec.repair(
|| Ok(Box::new(input.clone())),
|| Ok(Box::new(input.clone())),
cancel_signal,
)
.context("Failed to repair file")?;
Ok(())
}
pub fn fec_main(cli: &FecCli, cancel_signal: &AtomicBool) -> Result<()> {
match &cli.command {
FecCommand::Generate(c) => generate_subcommand(c, cancel_signal),
FecCommand::Verify(c) => verify_subcommand(c, cancel_signal),
FecCommand::Repair(c) => repair_subcommand(c, cancel_signal),
}
}
/// Generate FEC data for a file.
#[derive(Debug, Parser)]
struct GenerateCli {
/// Path to input data.
#[arg(short, long, value_name = "FILE", value_parser)]
input: PathBuf,
/// Path to output FEC data.
#[arg(short, long, value_name = "FILE", value_parser)]
fec: PathBuf,
/// Number of parity bytes per RS block (min 2, max 24).
#[arg(short, long, value_name = "BYTES", default_value = "2")]
parity: u8,
}
/// Verify that a file contains no errors.
#[derive(Debug, Parser)]
struct VerifyCli {
/// Path to input data.
#[arg(short, long, value_name = "FILE", value_parser)]
input: PathBuf,
/// Path to input FEC data.
#[arg(short, long, value_name = "FILE", value_parser)]
fec: PathBuf,
}
/// Repair a file.
#[derive(Debug, Parser)]
struct RepairCli {
/// Path to data.
///
/// The file will be modified in place.
#[arg(short, long, value_name = "FILE", value_parser)]
input: PathBuf,
/// Path to input FEC data.
#[arg(short, long, value_name = "FILE", value_parser)]
fec: PathBuf,
}
#[derive(Debug, Subcommand)]
enum FecCommand {
Generate(GenerateCli),
Verify(VerifyCli),
Repair(RepairCli),
}
/// Generate dm-verity FEC data and verify/repair files.
///
/// These commands operate on FEC files with AOSP's header format.
#[derive(Debug, Parser)]
pub struct FecCli {
#[command(subcommand)]
command: FecCommand,
}
avbroot/src/cli/mod.rs
+1
View File
@@ -7,6 +7,7 @@ pub mod args;
pub mod avb;
pub mod boot;
pub mod completion;
pub mod fec;
pub mod key;
pub mod ota;
pub mod ramdisk;
avbroot/src/cli/ota.rs
+1 -1
View File
@@ -382,7 +382,7 @@ fn update_vbmeta_descriptors(
(Descriptor::Hash(pd), Descriptor::Hash(d)) => {
*pd = d.clone();
}
(Descriptor::Hashtree(pd), Descriptor::Hashtree(d)) => {
(Descriptor::HashTree(pd), Descriptor::HashTree(d)) => {
*pd = d.clone();
}
_ => {
avbroot/src/format/avb.rs
+106 -61
View File
@@ -7,7 +7,7 @@ use std::{
cmp, fmt,
io::{self, Cursor, Read, Seek, SeekFrom, Write},
str,
sync::atomic::{AtomicBool, Ordering},
sync::atomic::AtomicBool,
};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
@@ -20,7 +20,10 @@ use sha2::{Digest, Sha256, Sha512};
use thiserror::Error;
use crate::{
format::padding,
format::{
fec::{self, Fec},
padding,
},
stream::{
self, CountingReader, FromReader, ReadDiscardExt, ReadSeek, ReadStringExt, ToWriter,
WriteStringExt, WriteZerosExt,
@@ -54,8 +57,6 @@ pub enum Error {
PaddingTooLong,
#[error("{0:?} field padding contains non-zero bytes")]
PaddingNotZero(&'static str),
#[error("{0:?} field + {1:?} field is out of bounds")]
OutOfBounds(&'static str, &'static str),
#[error("{0:?} field size does not equal size of contained items")]
IncorrectCombinedSize(&'static str),
#[error("Invalid VBMeta header magic: {0:?}")]
@@ -68,20 +69,31 @@ pub enum Error {
UnsupportedAlgorithm(AlgorithmType),
#[error("Hashing algorithm not supported: {0:?}")]
UnsupportedHashAlgorithm(String),
#[error("Incorrect key size ({0} bytes) for algorithm {1:?} ({2} bytes)")]
IncorrectKeySize(usize, AlgorithmType, usize),
#[error("Expected root digest {0}, but have {1}")]
InvalidRootDigest(String, String),
#[error("Expected hash tree {0}, but have {1}")]
InvalidHashtree(String, String),
#[error("Incorrect key size ({key_size} bytes) for algorithm {algo:?} ({} bytes)", algo.public_key_len())]
IncorrectKeySize {
key_size: usize,
algo: AlgorithmType,
},
#[error("Expected root digest {expected}, but have {actual}")]
InvalidRootDigest { expected: String, actual: String },
#[error("Expected hash tree {expected}, but have {actual}")]
InvalidHashTree { expected: String, actual: String },
#[error("Hash tree does not immediately follow image data")]
HashTreeGap,
#[error("FEC data does not immediately follow hash tree")]
FecDataGap,
#[error("FEC requires data block size ({data}) and hash block size ({hash}) to match")]
MismatchedFecBlockSizes { data: u32, hash: u32 },
#[error("Failed to RSA sign digest")]
RsaSignError(rsa::Error),
RsaSign(rsa::Error),
#[error("Failed to RSA verify signature")]
RsaVerifyError(rsa::Error),
RsaVerify(rsa::Error),
#[error("{0} byte image size is too small to fit header or footer")]
ImageSizeTooSmall(u64),
#[error("FEC error")]
Fec(#[from] fec::Error),
#[error("I/O error")]
IoError(#[from] io::Error),
Io(#[from] io::Error),
}
type Result<T> = std::result::Result<T, Error>;
@@ -172,11 +184,11 @@ impl AlgorithmType {
Self::None | Self::Unknown(_) => vec![],
Self::Sha256Rsa2048 | Self::Sha256Rsa4096 | Self::Sha256Rsa8192 => {
let scheme = Pkcs1v15Sign::new::<Sha256>();
key.sign(scheme, digest).map_err(Error::RsaSignError)?
key.sign(scheme, digest).map_err(Error::RsaSign)?
}
Self::Sha512Rsa2048 | Self::Sha512Rsa4096 | Self::Sha512Rsa8192 => {
let scheme = Pkcs1v15Sign::new::<Sha512>();
key.sign(scheme, digest).map_err(Error::RsaSignError)?
key.sign(scheme, digest).map_err(Error::RsaSign)?
}
};
@@ -189,12 +201,12 @@ impl AlgorithmType {
Self::Sha256Rsa2048 | Self::Sha256Rsa4096 | Self::Sha256Rsa8192 => {
let scheme = Pkcs1v15Sign::new::<Sha256>();
key.verify(scheme, digest, signature)
.map_err(Error::RsaVerifyError)?;
.map_err(Error::RsaVerify)?;
}
Self::Sha512Rsa2048 | Self::Sha512Rsa4096 | Self::Sha512Rsa8192 => {
let scheme = Pkcs1v15Sign::new::<Sha512>();
key.verify(scheme, digest, signature)
.map_err(Error::RsaVerifyError)?;
.map_err(Error::RsaVerify)?;
}
}
@@ -281,7 +293,7 @@ impl<W: Write> ToWriter<W> for PropertyDescriptor {
}
#[derive(Clone, Eq, PartialEq)]
pub struct HashtreeDescriptor {
pub struct HashTreeDescriptor {
pub dm_verity_version: u32,
pub image_size: u64,
pub tree_offset: u64,
@@ -299,9 +311,9 @@ pub struct HashtreeDescriptor {
pub reserved: [u8; 60],
}
impl fmt::Debug for HashtreeDescriptor {
impl fmt::Debug for HashTreeDescriptor {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("HashtreeDescriptor")
f.debug_struct("HashTreeDescriptor")
.field("dm_verity_version", &self.dm_verity_version)
.field("image_size", &self.image_size)
.field("tree_offset", &self.tree_offset)
@@ -321,7 +333,7 @@ impl fmt::Debug for HashtreeDescriptor {
}
}
impl HashtreeDescriptor {
impl HashTreeDescriptor {
/// Calculate the hash tree digests for a single level of the tree. If the
/// reader's position is block-aligned and `image_size` is a multiple of the
/// block size, then this function can also be used to calculate the digests
@@ -342,12 +354,7 @@ impl HashtreeDescriptor {
let mut result = vec![];
while image_size > 0 {
if cancel_signal.load(Ordering::SeqCst) {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
"Received cancel signal",
));
}
stream::check_cancel(cancel_signal)?;
let n = image_size.min(buf.len() as u64) as usize;
reader.read_exact(&mut buf[..n])?;
@@ -505,10 +512,14 @@ impl HashtreeDescriptor {
)?;
if self.root_digest != actual_root_digest {
return Err(Error::InvalidRootDigest(
hex::encode(&self.root_digest),
hex::encode(actual_root_digest),
));
return Err(Error::InvalidRootDigest {
expected: hex::encode(&self.root_digest),
actual: hex::encode(actual_root_digest),
});
}
if self.tree_offset != self.image_size {
return Err(Error::HashTreeGap);
}
let mut reader = open_input()?;
@@ -522,21 +533,55 @@ impl HashtreeDescriptor {
let expected = ring::digest::digest(algorithm, &hash_tree);
let actual = ring::digest::digest(algorithm, &actual_hash_tree);
return Err(Error::InvalidRootDigest(
hex::encode(expected),
hex::encode(actual),
));
return Err(Error::InvalidHashTree {
expected: hex::encode(expected),
actual: hex::encode(actual),
});
}
// The FEC data section is optional.
if self.fec_size != 0 {
if self.fec_offset != self.tree_offset + self.tree_size {
return Err(Error::FecDataGap);
} else if self.data_block_size != self.hash_block_size {
return Err(Error::MismatchedFecBlockSizes {
data: self.data_block_size,
hash: self.hash_block_size,
});
}
let fec_size = self
.fec_size
.to_usize()
.ok_or_else(|| Error::IntegerTooLarge("fec_size"))?;
let parity = self
.fec_num_roots
.to_u8()
.ok_or_else(|| Error::IntegerTooLarge("fec_num_roots"))?;
// The FEC covers the hash tree as well.
let fec = Fec::new(
self.tree_offset + self.tree_size,
self.data_block_size,
parity,
)?;
let mut fec_data = vec![0u8; fec_size];
// Already seeked to FEC.
reader.read_exact(&mut fec_data)?;
fec.verify(open_input, &fec_data, cancel_signal)?;
}
Ok(())
}
}
impl DescriptorTag for HashtreeDescriptor {
impl DescriptorTag for HashTreeDescriptor {
const TAG: u64 = 1;
}
impl<R: Read> FromReader<R> for HashtreeDescriptor {
impl<R: Read> FromReader<R> for HashTreeDescriptor {
type Error = Error;
fn from_reader(mut reader: R) -> Result<Self> {
@@ -598,7 +643,7 @@ impl<R: Read> FromReader<R> for HashtreeDescriptor {
}
}
impl<W: Write> ToWriter<W> for HashtreeDescriptor {
impl<W: Write> ToWriter<W> for HashTreeDescriptor {
type Error = Error;
fn to_writer(&self, mut writer: W) -> Result<()> {
@@ -701,10 +746,10 @@ impl HashDescriptor {
let digest = context.finish();
if self.root_digest != digest.as_ref() {
return Err(Error::InvalidRootDigest(
hex::encode(&self.root_digest),
hex::encode(digest),
));
return Err(Error::InvalidRootDigest {
expected: hex::encode(&self.root_digest),
actual: hex::encode(digest),
});
}
Ok(())
@@ -939,7 +984,7 @@ impl<W: Write> ToWriter<W> for ChainPartitionDescriptor {
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Descriptor {
Property(PropertyDescriptor),
Hashtree(HashtreeDescriptor),
HashTree(HashTreeDescriptor),
Hash(HashDescriptor),
KernelCmdline(KernelCmdlineDescriptor),
ChainPartition(ChainPartitionDescriptor),
@@ -949,7 +994,7 @@ pub enum Descriptor {
impl Descriptor {
pub fn partition_name(&self) -> Option<&str> {
match self {
Self::Hashtree(d) => Some(&d.partition_name),
Self::HashTree(d) => Some(&d.partition_name),
Self::Hash(d) => Some(&d.partition_name),
Self::ChainPartition(d) => Some(&d.partition_name),
_ => None,
@@ -971,9 +1016,9 @@ impl<R: Read> FromReader<R> for Descriptor {
let d = PropertyDescriptor::from_reader(&mut inner_reader)?;
Self::Property(d)
}
HashtreeDescriptor::TAG => {
let d = HashtreeDescriptor::from_reader(&mut inner_reader)?;
Self::Hashtree(d)
HashTreeDescriptor::TAG => {
let d = HashTreeDescriptor::from_reader(&mut inner_reader)?;
Self::HashTree(d)
}
HashDescriptor::TAG => {
let d = HashDescriptor::from_reader(&mut inner_reader)?;
@@ -1019,7 +1064,7 @@ impl<W: Write> ToWriter<W> for Descriptor {
d.to_writer(&mut inner_writer)?;
d.get_tag()
}
Self::Hashtree(d) => {
Self::HashTree(d) => {
d.to_writer(&mut inner_writer)?;
d.get_tag()
}
@@ -1210,11 +1255,10 @@ impl Header {
}
if key_raw.len() != self.algorithm_type.public_key_len() {
return Err(Error::IncorrectKeySize(
key_raw.len(),
self.algorithm_type,
self.algorithm_type.public_key_len(),
));
return Err(Error::IncorrectKeySize {
key_size: key_raw.len(),
algo: self.algorithm_type,
});
}
// The public key and the sizes of the hash and signature are included
@@ -1381,7 +1425,7 @@ impl<R: Read> FromReader<R> for Header {
.checked_add(hash_size)
.map_or(false, |s| s > auth_block.len())
{
return Err(Error::OutOfBounds("hash_offset", "hash_size"));
return Err(Error::IntegerTooLarge("hash_offset + hash_size"));
}
let hash = &auth_block[hash_offset..hash_offset + hash_size];
@@ -1389,7 +1433,7 @@ impl<R: Read> FromReader<R> for Header {
.checked_add(signature_size)
.map_or(false, |s| s > auth_block.len())
{
return Err(Error::OutOfBounds("signature_offset", "signature_size"));
return Err(Error::IntegerTooLarge("signature_offset + signature_size"));
}
let signature = &auth_block[signature_offset..signature_offset + signature_size];
@@ -1399,7 +1443,9 @@ impl<R: Read> FromReader<R> for Header {
.checked_add(public_key_size)
.map_or(false, |s| s > aux_block.len())
{
return Err(Error::OutOfBounds("public_key_offset", "public_key_size"));
return Err(Error::IntegerTooLarge(
"public_key_offset + public_key_size",
));
}
let public_key = &aux_block[public_key_offset..public_key_offset + public_key_size];
@@ -1407,9 +1453,8 @@ impl<R: Read> FromReader<R> for Header {
.checked_add(public_key_metadata_size)
.map_or(false, |s| s > aux_block.len())
{
return Err(Error::OutOfBounds(
"public_key_metadata_offset",
"public_key_metadata_size",
return Err(Error::IntegerTooLarge(
"public_key_metadata_offset + public_key_metadata_size",
));
}
let public_key_metadata = &aux_block
@@ -1582,7 +1627,7 @@ pub fn decode_public_key(data: &[u8]) -> Result<RsaPublicKey> {
let modulus = BigUint::from_bytes_be(&modulus_raw);
let public_key =
RsaPublicKey::new(modulus, BigUint::from(65537u32)).map_err(Error::RsaVerifyError)?;
RsaPublicKey::new(modulus, BigUint::from(65537u32)).map_err(Error::RsaVerify)?;
Ok(public_key)
}
@@ -1597,7 +1642,7 @@ pub fn load_image(mut reader: impl Read + Seek) -> Result<(Header, Option<Footer
let footer = match Footer::from_reader(&mut reader) {
Ok(f) => Some(f),
Err(e @ Error::IoError(_)) => return Err(e),
Err(e @ Error::Io(_)) => return Err(e),
Err(_) => None,
};
avbroot/src/format/fec.rs
+824
View File
@@ -0,0 +1,824 @@
/*
* SPDX-FileCopyrightText: 2023 Andrew Gunnerson
* SPDX-License-Identifier: GPL-3.0-only
*/
use std::{
fmt,
io::{self, Cursor, Read, Seek, SeekFrom, Write},
sync::atomic::AtomicBool,
};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use num_traits::ToPrimitive;
use rayon::{
prelude::{IndexedParallelIterator, ParallelIterator},
slice::{ParallelSlice, ParallelSliceMut},
};
use thiserror::Error;
use crate::{
format::verityrs,
stream::{self, FromReader, ReadSeek, ToWriter, WriteSeek, WriteZerosExt},
util::NumBytes,
};
// Not to be confused with the 255-byte RS block size.
const FEC_BLOCK_SIZE: usize = 4096;
const FEC_HEADER_SIZE: usize = 60;
const FEC_MAGIC: u32 = 0xFECFECFE;
const FEC_VERSION: u32 = 0;
#[derive(Debug, Error)]
pub enum Error {
#[error("FEC with parity byte count of {0} is not supported")]
UnsupportedParity(u8),
#[error("Cannot calculate FEC for empty data")]
InputEmpty,
#[error("Input size ({input}) is not a multiple of FEC block size ({block})")]
NotBlockAligned { input: u64, block: u32 },
#[error("FEC should have size {expected} for input size {input}, but has size {actual}")]
InvalidFecSize {
input: u64,
expected: usize,
actual: usize,
},
#[error("Cannot repair data due to too many errors")]
TooManyErrors,
#[error("Input data contains errors")]
HasErrors,
#[error("Data is too small to contain FEC headers")]
DataTooSmall,
#[error("The two FEC headers are different")]
HeadersDifferent,
#[error("Invalid FEC header magic: {0:#x}")]
InvalidHeaderMagic(u32),
#[error("Unsupported FEC header version: {0}")]
UnsupportedHeaderVersion(u32),
#[error("Invalid FEC header size: {0}")]
InvalidHeaderSize(u32),
#[error("FEC size in header {value} does not match available data ({available})")]
InvalidHeaderFecSize { value: usize, available: usize },
#[error("Expected FEC digest {expected}, but have {actual}")]
InvalidFecDigest { expected: String, actual: String },
#[error("{0:?} field exceeds integer bounds")]
IntegerTooLarge(&'static str),
#[error("I/O error")]
Io(#[from] io::Error),
}
type Result<T> = std::result::Result<T, Error>;
/// A small wrapper around a byte array to represent a single Reed-Solomon
/// codeword for any `RS(255, K)`.
struct Codeword {
data: [u8; 255],
rs_k: u8,
}
impl Codeword {
fn new(rs_k: u8) -> Self {
Self {
data: [0u8; 255],
rs_k,
}
}
fn data(&self) -> &[u8] {
&self.data[..usize::from(self.rs_k)]
}
fn data_mut(&mut self) -> &mut [u8] {
&mut self.data[..usize::from(self.rs_k)]
}
fn parity(&mut self) -> &[u8] {
&self.data[usize::from(self.rs_k)..]
}
fn parity_mut(&mut self) -> &mut [u8] {
&mut self.data[usize::from(self.rs_k)..]
}
fn all(&self) -> &[u8] {
&self.data
}
fn all_mut(&mut self) -> &mut [u8] {
&mut self.data
}
}
/// Since Rust's built-in .div_ceil() is still nightly-only.
fn div_ceil(dividend: u64, divisor: u64) -> u64 {
dividend / divisor + u64::from(dividend % divisor != 0)
}
/// A type for performing FEC generation, verification, and error correction for
/// a specific file size and Reed Solomon configuration. The implementation uses
/// dm-verity's interleaving access pattern.
///
/// The interleaving access pattern can be visualized by placing the file
/// offsets in a two-dimensional grid. For example, when reading a 2072576-byte
/// file for calculating RS(255, 253):
///
/// ```text
/// | <-------- Round 0 --------> | <-------- Round 1 --------> |
/// |-----------------------------|-----------------------------|
/// ^ | 0 1 ... 4095 | 4096 4097 ... 8191 |
/// | | 8192 8192 ... 12287 | 12288 12289 ... 16383 |
/// rs_k | 16384 16385 ... 20479 | 20480 20481 ... 24575 |
/// | | ....... ....... ... ....... | ....... ....... ... ....... |
/// v | 2064384 2064385 ... 2068479 | 2068480 2068481 ... 2072575 |
/// ```
///
/// A regular sequential read of the file is traversing the grid row-by-row,
/// while an interleaving read is traversing the grid column-by-column. Each
/// column is always `rs_k` items tall, so each column forms the data portion of
/// an RS codeword. The number of columns is always a multiple of the FEC block
/// size. Since RS operates on fixed-size codewords and a file size might not
/// always fill the grid completely, out-of-bounds offsets are treated as if
/// they contain a `\0` byte.
///
/// All operations are multithreaded with I/O operations parallelized at the
/// "round" level and RS operations parallelized at the column level.
pub struct Fec {
file_size: u64,
block_size: u32,
rs_k: u8,
rounds: u64,
}
impl Fec {
pub fn new(file_size: u64, block_size: u32, parity: u8) -> Result<Self> {
if file_size == 0 {
return Err(Error::InputEmpty);
} else if file_size % u64::from(block_size) != 0 {
return Err(Error::NotBlockAligned {
input: file_size,
block: block_size,
});
}
let rs_k = 255 - parity;
if !verityrs::FN_ENCODE.contains_key(&rs_k) {
return Err(Error::UnsupportedParity(parity));
}
let blocks = div_ceil(file_size, u64::from(block_size));
let rounds = div_ceil(blocks, u64::from(rs_k));
Ok(Self {
file_size,
block_size,
rs_k,
rounds,
})
}
/// Get the number of parity bytes per codeword.
#[inline]
fn parity(&self) -> u8 {
255 - self.rs_k
}
/// Get the size of the FEC data needed to cover the entire file.
#[inline]
fn fec_size(&self) -> usize {
usize::from(self.parity()) * self.rounds as usize * self.block_size as usize
}
/// Get the backing file offset for the specified `offset` in the
/// interleaved view.
fn backing_offset(&self, offset: u64) -> u64 {
let rs_k = u64::from(self.rs_k);
offset / rs_k + offset % rs_k * self.rounds * u64::from(self.block_size)
}
/// Read a raw sequential block from the backing file, starting at offset
/// `offset` in the interleaved view. This reads a horizontal block-aligned
/// slice in the file offset grid.
fn read_seq_block(
&self,
mut reader: impl Read + Seek,
offset: u64,
buf: &mut [u8],
) -> io::Result<()> {
assert_eq!(
buf.len(),
self.block_size as usize,
"Buffer does not match block size",
);
let backing_offset = self.backing_offset(offset);
// Out of bounds offsets are treated as if they contain zeros.
if backing_offset >= self.file_size {
buf.fill(0);
} else {
reader.seek(SeekFrom::Start(backing_offset))?;
reader.read_exact(buf)?;
}
Ok(())
}
/// Write a raw sequential block to the backing file, starting at offset
/// `offset` in the interleaved view. This writes a horizontal block-aligned
/// slice in the file offset grid.
fn write_seq_block(
&self,
mut writer: impl Write + Seek,
offset: u64,
buf: &[u8],
) -> io::Result<()> {
assert_eq!(
buf.len(),
self.block_size as usize,
"Buffer does not match block size",
);
let backing_offset = self.backing_offset(offset);
// Out of bounds offsets are ignored.
if backing_offset < self.file_size {
writer.seek(SeekFrom::Start(backing_offset))?;
writer.write_all(buf)?;
}
Ok(())
}
/// Read the nth round from the file. The data is laid out sequentially
/// (row-by-row).
fn read_round(&self, mut reader: impl Read + Seek, round: u64) -> io::Result<Vec<u8>> {
let mut grid = vec![0u8; usize::from(self.rs_k) * self.block_size as usize];
for row in 0..self.rs_k {
let interleaved_offset =
round * u64::from(self.rs_k) * u64::from(self.block_size) + u64::from(row);
let row_start = usize::from(row) * self.block_size as usize;
let row_end = row_start + self.block_size as usize;
let row_slice = &mut grid[row_start..row_end];
self.read_seq_block(&mut reader, interleaved_offset, row_slice)?;
}
Ok(grid)
}
/// Write the nth round to the file. The data is expected to be laid out
/// sequentially (row-by-row).
fn write_round(
&self,
mut writer: impl Write + Seek,
round: u64,
grid: &[u8],
) -> io::Result<()> {
for row in 0..self.rs_k {
let interleaved_offset =
round * u64::from(self.rs_k) * u64::from(self.block_size) + u64::from(row);
let row_start = usize::from(row) * self.block_size as usize;
let row_end = row_start + self.block_size as usize;
let row_slice = &grid[row_start..row_end];
self.write_seq_block(&mut writer, interleaved_offset, row_slice)?;
}
Ok(())
}
/// Get the nth RS codeword from a round's grid.
fn get_codeword(&self, grid: &[u8], column: usize) -> Codeword {
let mut codeword = Codeword::new(self.rs_k);
let data = codeword.data_mut();
for row in 0..usize::from(self.rs_k) {
data[row] = grid[row * self.block_size as usize + column];
}
codeword
}
/// Put the nth RS codeword into a round's grid.
fn put_codeword(&self, grid: &mut [u8], column: usize, codeword: &Codeword) {
let data = codeword.data();
for row in 0..usize::from(self.rs_k) {
grid[row * self.block_size as usize + column] = data[row];
}
}
/// Generate FEC data for a single round.
fn generate_one_round(
&self,
reader: impl Read + Seek,
round: u64,
fec: &mut [u8],
) -> Result<()> {
assert_eq!(
fec.len(),
usize::from(self.parity()) * self.block_size as usize,
"FEC buffer length does not match block size",
);
let grid = self.read_round(reader, round)?;
let encode = verityrs::FN_ENCODE[&self.rs_k];
let parity = usize::from(self.parity());
for (column, buf) in fec.chunks_exact_mut(parity).enumerate() {
let mut codeword = self.get_codeword(&grid, column);
encode(codeword.all_mut());
buf.copy_from_slice(codeword.parity());
}
Ok(())
}
/// Verify file data for a single round.
fn verify_one_round(&self, reader: impl Read + Seek, round: u64, fec: &[u8]) -> Result<()> {
assert_eq!(
fec.len(),
usize::from(self.parity()) * self.block_size as usize,
"FEC buffer length does not match block size",
);
let grid = self.read_round(reader, round)?;
let is_correct = verityrs::FN_IS_CORRECT[&self.rs_k];
let parity = usize::from(self.parity());
for (column, buf) in fec.chunks_exact(parity).enumerate() {
let mut codeword = self.get_codeword(&grid, column);
codeword.parity_mut().copy_from_slice(buf);
if !is_correct(codeword.all()) {
return Err(Error::HasErrors);
}
}
Ok(())
}
/// Repair file data for a single round.
fn repair_one_round(
&self,
reader: impl Read + Seek,
writer: impl Write + Seek,
round: u64,
fec: &[u8],
) -> Result<u64> {
assert_eq!(
fec.len(),
usize::from(self.parity()) * self.block_size as usize,
"FEC buffer length does not match block size",
);
let mut grid = self.read_round(reader, round)?;
let correct_errors = verityrs::FN_CORRECT_ERRORS[&self.rs_k];
let parity = usize::from(self.parity());
let mut num_corrected = 0;
for (column, buf) in fec.chunks_exact(parity).enumerate() {
let mut codeword = self.get_codeword(&grid, column);
codeword.parity_mut().copy_from_slice(buf);
let n = correct_errors(codeword.all_mut()).ok_or(Error::TooManyErrors)?;
if n > 0 {
self.put_codeword(&mut grid, column, &codeword);
}
num_corrected += n as u64;
}
if num_corrected > 0 {
self.write_round(writer, round, &grid)?;
}
Ok(num_corrected)
}
/// Generate FEC data for the file. The file size must match the file size
/// given to [`Self::new()`].
///
/// This function is multithreaded and uses rayon's global thread pool.
pub fn generate(
&self,
open_input: impl Fn() -> io::Result<Box<dyn ReadSeek>> + Sync,
cancel_signal: &AtomicBool,
) -> Result<Vec<u8>> {
let fec_size = self.fec_size();
let mut fec = vec![0u8; fec_size];
fec.par_chunks_exact_mut(fec_size / self.rounds as usize)
.enumerate()
.map(|(round, buf)| -> Result<()> {
stream::check_cancel(cancel_signal)?;
let reader = open_input()?;
self.generate_one_round(reader, round as u64, buf)
})
.collect::<Result<()>>()?;
Ok(fec)
}
/// Verify that the file contains no errors. This is significantly faster
/// than [`Self::repair()`] if only error detection, not correction, is
/// needed.
///
/// This function is multithreaded and uses rayon's global thread pool.
pub fn verify(
&self,
open_input: impl Fn() -> io::Result<Box<dyn ReadSeek>> + Sync,
fec: &[u8],
cancel_signal: &AtomicBool,
) -> Result<()> {
let fec_size = self.fec_size();
if fec.len() != fec_size {
return Err(Error::InvalidFecSize {
input: self.file_size,
expected: fec_size,
actual: fec.len(),
});
}
fec.par_chunks_exact(fec_size / self.rounds as usize)
.enumerate()
.map(|(round, buf)| -> Result<()> {
stream::check_cancel(cancel_signal)?;
let reader = open_input()?;
self.verify_one_round(reader, round as u64, buf)
})
.collect::<Result<()>>()?;
Ok(())
}
/// Repair the file. Up to `parity / 2` bytes per codeword can be repaired.
/// If the file is successfully repaired, the number of repaired bytes is
/// returned. If the file is corrupt beyond repair, [`Error::TooManyErrors`]
/// is returned. It's not safe to assume that as much data as possible has
/// been repaired when [`Error::TooManyErrors`] is returned due to fail-fast
/// behavior.
///
/// This function corrects errors at unknown locations only. Correcting
/// erasures at known locations is not supported.
///
/// This function is multithreaded and uses rayon's global thread pool.
pub fn repair(
&self,
open_input: impl Fn() -> io::Result<Box<dyn ReadSeek>> + Sync,
open_output: impl Fn() -> io::Result<Box<dyn WriteSeek>> + Sync,
fec: &[u8],
cancel_signal: &AtomicBool,
) -> Result<u64> {
let fec_size = self.fec_size();
if fec.len() != fec_size {
return Err(Error::InvalidFecSize {
input: self.file_size,
expected: fec_size,
actual: fec.len(),
});
}
let num_corrected = fec
.par_chunks_exact(fec_size / self.rounds as usize)
.enumerate()
.map(|(round, buf)| -> Result<u64> {
stream::check_cancel(cancel_signal)?;
let reader = open_input()?;
let writer = open_output()?;
self.repair_one_round(reader, writer, round as u64, buf)
})
.collect::<Result<Vec<u64>>>()?
.into_iter()
.sum();
Ok(num_corrected)
}
}
/// A type for reading and writing AOSP's standalone FEC image format.
///
/// The FEC data parser in this implementation is strict. All header fields,
/// like the version, header size, and digest, must be valid and both copies of
/// the header must match.
#[derive(Clone, PartialEq, Eq)]
pub struct FecImage {
pub fec: Vec<u8>,
pub data_size: u64,
pub parity: u8,
}
impl fmt::Debug for FecImage {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Fec")
.field("fec", &NumBytes(self.fec.len()))
.field("data_size", &self.data_size)
.field("parity", &self.parity)
.finish()
}
}
impl FecImage {
/// Generate FEC data for a file. `parity` is the number of parity bytes per
/// 255-byte Reed-Solomon codeword.
pub fn generate(
open_input: impl Fn() -> io::Result<Box<dyn ReadSeek>> + Sync,
parity: u8,
cancel_signal: &AtomicBool,
) -> Result<Self> {
let data_size = {
let mut file = open_input()?;
file.seek(SeekFrom::End(0))?
};
let fec = Fec::new(data_size, FEC_BLOCK_SIZE as u32, parity)?;
let fec_data = fec.generate(open_input, cancel_signal)?;
Ok(Self {
fec: fec_data,
data_size,
parity,
})
}
/// Check that a file contains no errors. This is significantly faster than
/// [`Self::repair()`] if performing a repair is not necessary.
pub fn verify(
&self,
open_input: impl Fn() -> io::Result<Box<dyn ReadSeek>> + Sync,
cancel_signal: &AtomicBool,
) -> Result<()> {
let fec = Fec::new(self.data_size, FEC_BLOCK_SIZE as u32, self.parity)?;
fec.verify(open_input, &self.fec, cancel_signal)
}
/// Repair a file using this instance's FEC data. The maximum correctable
/// errors per 255-byte codeword is half of [`Self::parity`]. Returns the
/// number of bytes corrected if the file is successfully repaired or
/// [`Error::TooManyErrors`] if the file cannot be repaired. This function
/// fails fast. If an RS codeword cannot be repaired, other potentially
/// repairable codewords may not be repaired.
///
/// Note that if there are too many errors inside a certain codeword, it's
/// possible for there to be a false positive where the corrupted codeword
/// is "corrected" into an incorrect value. FEC error detection is not a
/// replacement for cryptographically secure digests.
///
/// The inputs and outputs should point to the same underlying file because
/// only regions where errors are corrected are written. It is guaranteed
/// that multiple threads will always read and write disjoint file offsets.
pub fn repair(
&self,
open_input: impl Fn() -> io::Result<Box<dyn ReadSeek>> + Sync,
open_output: impl Fn() -> io::Result<Box<dyn WriteSeek>> + Sync,
cancel_signal: &AtomicBool,
) -> Result<u64> {
let fec = Fec::new(self.data_size, FEC_BLOCK_SIZE as u32, self.parity)?;
fec.repair(open_input, open_output, &self.fec, cancel_signal)
}
/// Build one instance of the FEC header. The caller is responsible for
/// writing it to both of the header locations at the end of the file.
fn build_header(&self) -> Result<[u8; FEC_HEADER_SIZE]> {
let fec_size = self
.fec
.len()
.to_u32()
.ok_or_else(|| Error::IntegerTooLarge("fec_size"))?;
let mut writer = Cursor::new([0u8; FEC_HEADER_SIZE]);
let digest = ring::digest::digest(&ring::digest::SHA256, &self.fec);
writer.write_u32::<LittleEndian>(FEC_MAGIC)?;
writer.write_u32::<LittleEndian>(FEC_VERSION)?;
writer.write_u32::<LittleEndian>(FEC_HEADER_SIZE as u32)?;
writer.write_u32::<LittleEndian>(self.parity.into())?;
writer.write_u32::<LittleEndian>(fec_size)?;
writer.write_u64::<LittleEndian>(self.data_size)?;
writer.write_all(digest.as_ref())?;
Ok(writer.into_inner())
}
}
impl<R: Read> FromReader<R> for FecImage {
type Error = Error;
fn from_reader(mut reader: R) -> Result<Self> {
// Avoid requiring seekable readers since we need to read everything
// into memory anyway.
let mut fec = Vec::new();
reader.read_to_end(&mut fec)?;
if fec.len() < FEC_BLOCK_SIZE {
return Err(Error::DataTooSmall);
}
// Make sure both headers match.
let header1_offset = fec.len() - FEC_BLOCK_SIZE;
let header2_offset = fec.len() - FEC_HEADER_SIZE;
let header1_raw = &fec[header1_offset..header1_offset + FEC_HEADER_SIZE];
let header2_raw = &fec[header2_offset..header2_offset + FEC_HEADER_SIZE];
if header1_raw != header2_raw {
return Err(Error::HeadersDifferent);
}
let mut header_reader = Cursor::new(header1_raw);
let magic = header_reader.read_u32::<LittleEndian>()?;
if magic != FEC_MAGIC {
return Err(Error::InvalidHeaderMagic(magic));
}
let version = header_reader.read_u32::<LittleEndian>()?;
if version != FEC_VERSION {
return Err(Error::UnsupportedHeaderVersion(version));
}
let header_size = header_reader.read_u32::<LittleEndian>()?;
if header_size != FEC_HEADER_SIZE as u32 {
return Err(Error::InvalidHeaderSize(header_size));
}
let parity = header_reader
.read_u32::<LittleEndian>()?
.to_u8()
.ok_or_else(|| Error::IntegerTooLarge("parity"))?;
let fec_size = header_reader
.read_u32::<LittleEndian>()?
.to_usize()
.ok_or_else(|| Error::IntegerTooLarge("fec_size"))?;
let actual_fec_size = fec.len() - FEC_BLOCK_SIZE;
if fec_size != actual_fec_size {
return Err(Error::InvalidHeaderFecSize {
value: fec_size,
available: actual_fec_size,
});
}
let input_size = header_reader.read_u64::<LittleEndian>()?;
let mut digest = [0u8; 32];
header_reader.read_exact(&mut digest)?;
// Chop off headers.
fec.resize(fec_size, 0);
let actual_digest = ring::digest::digest(&ring::digest::SHA256, &fec);
if digest != actual_digest.as_ref() {
return Err(Error::InvalidFecDigest {
expected: hex::encode(digest),
actual: hex::encode(actual_digest),
});
}
Ok(Self {
fec,
data_size: input_size,
parity,
})
}
}
impl<W: Write> ToWriter<W> for FecImage {
type Error = Error;
fn to_writer(&self, mut writer: W) -> Result<()> {
let header = self.build_header()?;
writer.write_all(&self.fec)?;
writer.write_all(&header)?;
writer.write_zeros_exact((FEC_BLOCK_SIZE - 2 * FEC_HEADER_SIZE) as u64)?;
writer.write_all(&header)?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::{
io::Seek,
sync::{atomic::AtomicBool, Arc},
};
use assert_matches::assert_matches;
use rand::RngCore;
use crate::stream::SharedCursor;
use super::*;
fn corrupt_byte(file: &mut SharedCursor, offset: u64) {
let mut buf = [0u8; 1];
file.seek(SeekFrom::Start(offset)).unwrap();
file.read_exact(&mut buf).unwrap();
buf[0] = buf[0].wrapping_add(1);
file.seek(SeekFrom::Start(offset)).unwrap();
file.write_all(&buf).unwrap();
}
fn run_test(block_size: u32, rs_k: u8) {
let cancel_signal = Arc::new(AtomicBool::new(false));
let parity = 255 - rs_k;
// Generate data big enough to span multiple rounds, but don't fill the
// offset grid to ensure that the out-of-bounds-is-0 behavior works.
let size = usize::from(rs_k) * block_size as usize * 3 - block_size as usize;
let mut file = SharedCursor::default();
let orig_digest = {
let mut buf = vec![0u8; size];
rand::thread_rng().fill_bytes(&mut buf);
file.write_all(&buf).unwrap();
ring::digest::digest(&ring::digest::SHA256, &buf)
};
let fec = Fec::new(size as u64, block_size, parity).unwrap();
assert_eq!(fec.rounds, 3);
let num_codewords = fec.rounds as usize * block_size as usize;
// Generate FEC data.
let fec_data = fec
.generate(|| Ok(Box::new(file.clone())), &cancel_signal)
.unwrap();
// Verify that there are no errors.
fec.verify(|| Ok(Box::new(file.clone())), &fec_data, &cancel_signal)
.unwrap();
// Verify that errors are detected.
corrupt_byte(&mut file, 0);
assert_matches!(
fec.verify(|| Ok(Box::new(file.clone())), &fec_data, &cancel_signal,),
Err(Error::HasErrors)
);
// Corrupt one byte in every single codeword.
for offset in 1..num_codewords {
corrupt_byte(&mut file, offset as u64);
}
// Verify that all the single-byte errors can be fixed.
fec.repair(
|| Ok(Box::new(file.clone())),
|| Ok(Box::new(file.clone())),
&fec_data,
&cancel_signal,
)
.unwrap();
let repaired_digest = {
let mut buf = Vec::new();
file.rewind().unwrap();
file.read_to_end(&mut buf).unwrap();
ring::digest::digest(&ring::digest::SHA256, &buf)
};
assert_eq!(repaired_digest.as_ref(), orig_digest.as_ref());
// We don't test for Error::TooManyErrors because of the chance of false
// positives due to the nature of RS.
}
#[test]
fn generate_verify_repair() {
for block_size in [1, 2, 4, 8, 16, 32, 64] {
for rs_k in verityrs::FN_ENCODE.keys() {
println!("Testing block_size={block_size}, rs_k={rs_k}");
run_test(block_size, *rs_k);
}
}
}
#[test]
fn round_trip_image() {
let cancel_signal = Arc::new(AtomicBool::new(false));
let mut file = SharedCursor::default();
{
let mut buf = [0u8; FEC_BLOCK_SIZE];
rand::thread_rng().fill_bytes(&mut buf);
file.write_all(&buf).unwrap();
}
let image = FecImage::generate(|| Ok(Box::new(file.clone())), 2, &cancel_signal).unwrap();
let mut fec_file = Cursor::new(Vec::new());
image.to_writer(&mut fec_file).unwrap();
fec_file.rewind().unwrap();
let new_image = FecImage::from_reader(&mut fec_file).unwrap();
assert_eq!(image, new_image);
}
}
avbroot/src/format/mod.rs
+2
View File
@@ -7,6 +7,8 @@ pub mod avb;
pub mod bootimage;
pub mod compression;
pub mod cpio;
pub mod fec;
pub mod ota;
pub mod padding;
pub mod payload;
pub mod verityrs;
avbroot/src/format/verityrs.rs
+142
View File
@@ -0,0 +1,142 @@
/*
* SPDX-FileCopyrightText: 2023 Andrew Gunnerson
* SPDX-License-Identifier: GPL-3.0-only
*/
// The gf256 library uses compile-time proc macro code generation. Since
// dm-verity supports RS(255, 231) through RS(255, 253), we'll generate RS
// implementations for every supported configuration.
#![allow(non_snake_case)]
use gf256::rs::rs;
use phf::phf_map;
#[rs(block = 255, data = 231)]
mod rs255w231 {}
#[rs(block = 255, data = 232)]
mod rs255w232 {}
#[rs(block = 255, data = 233)]
mod rs255w233 {}
#[rs(block = 255, data = 234)]
mod rs255w234 {}
#[rs(block = 255, data = 235)]
mod rs255w235 {}
#[rs(block = 255, data = 236)]
mod rs255w236 {}
#[rs(block = 255, data = 237)]
mod rs255w237 {}
#[rs(block = 255, data = 238)]
mod rs255w238 {}
#[rs(block = 255, data = 239)]
mod rs255w239 {}
#[rs(block = 255, data = 240)]
mod rs255w240 {}
#[rs(block = 255, data = 241)]
mod rs255w241 {}
#[rs(block = 255, data = 242)]
mod rs255w242 {}
#[rs(block = 255, data = 243)]
mod rs255w243 {}
#[rs(block = 255, data = 244)]
mod rs255w244 {}
#[rs(block = 255, data = 245)]
mod rs255w245 {}
#[rs(block = 255, data = 246)]
mod rs255w246 {}
#[rs(block = 255, data = 247)]
mod rs255w247 {}
#[rs(block = 255, data = 248)]
mod rs255w248 {}
#[rs(block = 255, data = 249)]
mod rs255w249 {}
#[rs(block = 255, data = 250)]
mod rs255w250 {}
#[rs(block = 255, data = 251)]
mod rs255w251 {}
#[rs(block = 255, data = 252)]
mod rs255w252 {}
#[rs(block = 255, data = 253)]
mod rs255w253 {}
pub static FN_ENCODE: phf::Map<u8, fn(&mut [u8])> = phf_map! {
231u8 => rs255w231::encode,
232u8 => rs255w232::encode,
233u8 => rs255w233::encode,
234u8 => rs255w234::encode,
235u8 => rs255w235::encode,
236u8 => rs255w236::encode,
237u8 => rs255w237::encode,
238u8 => rs255w238::encode,
239u8 => rs255w239::encode,
240u8 => rs255w240::encode,
241u8 => rs255w241::encode,
242u8 => rs255w242::encode,
243u8 => rs255w243::encode,
244u8 => rs255w244::encode,
245u8 => rs255w245::encode,
246u8 => rs255w246::encode,
247u8 => rs255w247::encode,
248u8 => rs255w248::encode,
249u8 => rs255w249::encode,
250u8 => rs255w250::encode,
251u8 => rs255w251::encode,
252u8 => rs255w252::encode,
253u8 => rs255w253::encode,
};
pub static FN_IS_CORRECT: phf::Map<u8, fn(&[u8]) -> bool> = phf_map! {
231u8 => rs255w231::is_correct,
232u8 => rs255w232::is_correct,
233u8 => rs255w233::is_correct,
234u8 => rs255w234::is_correct,
235u8 => rs255w235::is_correct,
236u8 => rs255w236::is_correct,
237u8 => rs255w237::is_correct,
238u8 => rs255w238::is_correct,
239u8 => rs255w239::is_correct,
240u8 => rs255w240::is_correct,
241u8 => rs255w241::is_correct,
242u8 => rs255w242::is_correct,
243u8 => rs255w243::is_correct,
244u8 => rs255w244::is_correct,
245u8 => rs255w245::is_correct,
246u8 => rs255w246::is_correct,
247u8 => rs255w247::is_correct,
248u8 => rs255w248::is_correct,
249u8 => rs255w249::is_correct,
250u8 => rs255w250::is_correct,
251u8 => rs255w251::is_correct,
252u8 => rs255w252::is_correct,
253u8 => rs255w253::is_correct,
};
// Each one of these has its own error type, but the functions can only fail one
// way (too many corrupt bytes), so just throw away the error and return an
// Option instead.
#[allow(clippy::type_complexity)]
pub static FN_CORRECT_ERRORS: phf::Map<u8, fn(&mut [u8]) -> Option<usize>> = phf_map! {
231u8 => |data: &mut [u8]| rs255w231::correct_errors(data).ok(),
232u8 => |data: &mut [u8]| rs255w232::correct_errors(data).ok(),
233u8 => |data: &mut [u8]| rs255w233::correct_errors(data).ok(),
234u8 => |data: &mut [u8]| rs255w234::correct_errors(data).ok(),
235u8 => |data: &mut [u8]| rs255w235::correct_errors(data).ok(),
236u8 => |data: &mut [u8]| rs255w236::correct_errors(data).ok(),
237u8 => |data: &mut [u8]| rs255w237::correct_errors(data).ok(),
238u8 => |data: &mut [u8]| rs255w238::correct_errors(data).ok(),
239u8 => |data: &mut [u8]| rs255w239::correct_errors(data).ok(),
240u8 => |data: &mut [u8]| rs255w240::correct_errors(data).ok(),
241u8 => |data: &mut [u8]| rs255w241::correct_errors(data).ok(),
242u8 => |data: &mut [u8]| rs255w242::correct_errors(data).ok(),
243u8 => |data: &mut [u8]| rs255w243::correct_errors(data).ok(),
244u8 => |data: &mut [u8]| rs255w244::correct_errors(data).ok(),
245u8 => |data: &mut [u8]| rs255w245::correct_errors(data).ok(),
246u8 => |data: &mut [u8]| rs255w246::correct_errors(data).ok(),
247u8 => |data: &mut [u8]| rs255w247::correct_errors(data).ok(),
248u8 => |data: &mut [u8]| rs255w248::correct_errors(data).ok(),
249u8 => |data: &mut [u8]| rs255w249::correct_errors(data).ok(),
250u8 => |data: &mut [u8]| rs255w250::correct_errors(data).ok(),
251u8 => |data: &mut [u8]| rs255w251::correct_errors(data).ok(),
252u8 => |data: &mut [u8]| rs255w252::correct_errors(data).ok(),
253u8 => |data: &mut [u8]| rs255w253::correct_errors(data).ok(),
};
avbroot/src/stream.rs
+17 -12
View File
@@ -560,6 +560,21 @@ impl Seek for SharedCursor {
}
}
/// Returns an I/O error with the [`io::ErrorKind::Interrupted`] type if
/// `cancel_signal` is true. This should be called frequently in I/O loops for
/// cancellation to be responsive.
#[inline]
pub fn check_cancel(cancel_signal: &AtomicBool) -> io::Result<()> {
if cancel_signal.load(Ordering::SeqCst) {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
"Received cancel signal",
));
}
Ok(())
}
/// Copy exactly `size` bytes from `reader` to `writer`, invoking `inspect`
/// after every buffer read iteration. If either `reader` or `writer` reaches
/// EOF before `size` bytes are copied, an error is returned. The operation is
@@ -574,12 +589,7 @@ pub fn copy_n_inspect(
let mut buf = [0u8; 16384];
while size > 0 {
if cancel_signal.load(Ordering::SeqCst) {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
"Received cancel signal",
));
}
check_cancel(cancel_signal)?;
let to_read = size.min(buf.len() as u64) as usize;
reader.read_exact(&mut buf[..to_read])?;
@@ -616,12 +626,7 @@ pub fn copy(
let mut copied = 0;
loop {
if cancel_signal.load(Ordering::SeqCst) {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
"Received cancel signal",
));
}
check_cancel(cancel_signal)?;
let n = reader.read(&mut buf)?;
if n == 0 {