awesome-android/avbroot
3
0
Fork 0
mirror of https://github.com/chenxiaolong/avbroot.git synced 2026-06-02 06:23:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add streaming CPIO reader and writer

Browse Source 
The Magisk boot image patcher still reads all the entries into memory
for simplicity, but everything else now uses the streaming reader.

Signed-off-by: Andrew Gunnerson <accounts+github@chiller3.com>
This commit is contained in:
Andrew Gunnerson
2023-09-29 17:39:53 -04:00
parent 61129e8ec7
commit 164274eb97
10 changed files with 621 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
avbroot/src/boot.rs
+65 -54
View File
@@ -32,7 +32,7 @@ use crate::{
avb::{self, Descriptor},
bootimage::{self, BootImage, BootImageExt, RamdiskMeta},
compression::{self, CompressedFormat, CompressedReader, CompressedWriter},
cpio::{self, CpioEntryNew},
cpio::{self, CpioEntry, CpioEntryData},
},
stream::{self, FromReader, HashingWriter, SectionReader, ToWriter},
};
@@ -73,18 +73,25 @@ pub enum Error {
type Result<T> = std::result::Result<T, Error>;
fn load_ramdisk(data: &[u8]) -> Result<(Vec<CpioEntryNew>, CompressedFormat)> {
fn load_ramdisk(
data: &[u8],
cancel_signal: &AtomicBool,
) -> Result<(Vec<CpioEntry>, CompressedFormat)> {
let raw_reader = Cursor::new(data);
let mut reader = CompressedReader::new(raw_reader, false)?;
let entries = cpio::load(&mut reader, false)?;
let entries = cpio::load(&mut reader, false, cancel_signal)?;
Ok((entries, reader.format()))
}
fn save_ramdisk(entries: &[CpioEntryNew], format: CompressedFormat) -> Result<Vec<u8>> {
fn save_ramdisk(
entries: &[CpioEntry],
format: CompressedFormat,
cancel_signal: &AtomicBool,
) -> Result<Vec<u8>> {
let raw_writer = Cursor::new(vec![]);
let mut writer = CompressedWriter::new(raw_writer, format)?;
cpio::save(&mut writer, entries, false)?;
cpio::save(&mut writer, entries, false, cancel_signal)?;
let raw_writer = writer.finish()?;
Ok(raw_writer.into_inner())
@@ -193,7 +200,7 @@ impl MagiskRootPatcher {
/// entries as `.backup/<path>`.
///
/// Both lists and entries within the lists may be mutated.
fn apply_magisk_backup(old_entries: &mut [CpioEntryNew], new_entries: &mut Vec<CpioEntryNew>) {
fn apply_magisk_backup(old_entries: &mut [CpioEntry], new_entries: &mut Vec<CpioEntry>) {
cpio::sort(old_entries);
cpio::sort(new_entries);
@@ -205,20 +212,20 @@ impl MagiskRootPatcher {
loop {
match (old_iter.peek(), new_iter.peek()) {
(Some(&old), Some(&new)) => match old.name.cmp(&new.name) {
(Some(&old), Some(&new)) => match old.path.cmp(&new.path) {
Ordering::Less => {
to_back_up.push(old);
old_iter.next();
}
Ordering::Equal => {
if old.content != new.content {
if old.data != new.data {
to_back_up.push(old);
}
old_iter.next();
new_iter.next();
}
Ordering::Greater => {
rm_list.extend(&new.name);
rm_list.extend(&new.path);
rm_list.push(b'\0');
new_iter.next();
}
@@ -228,7 +235,7 @@ impl MagiskRootPatcher {
old_iter.next();
}
(None, Some(new)) => {
rm_list.extend(&new.name);
rm_list.extend(&new.path);
rm_list.push(b'\0');
new_iter.next();
}
@@ -236,22 +243,20 @@ impl MagiskRootPatcher {
}
}
// Intentially using 000 permissions to match Magisk.
new_entries.push(CpioEntryNew::new_directory(b".backup"));
new_entries.push(CpioEntry::new_directory(b".backup", 0));
for old_entry in to_back_up {
let mut new_entry = old_entry.clone();
new_entry.name = b".backup/".to_vec();
new_entry.name.extend(&old_entry.name);
new_entry.path = b".backup/".to_vec();
new_entry.path.extend(&old_entry.path);
new_entries.push(new_entry);
}
{
// Intentially using 000 permissions to match Magisk.
let mut entry = CpioEntryNew::new_file(b".backup/.rmlist");
entry.content = rm_list;
new_entries.push(entry);
}
new_entries.push(CpioEntry::new_file(
b".backup/.rmlist",
0,
CpioEntryData::Data(rm_list),
));
}
}
@@ -269,7 +274,7 @@ impl BootImagePatcher for MagiskRootPatcher {
BootImage::VendorV3Through4(b) => b.ramdisks.first(),
};
let (mut entries, ramdisk_format) = match ramdisk {
Some(r) if !r.is_empty() => load_ramdisk(r)?,
Some(r) if !r.is_empty() => load_ramdisk(r, cancel_signal)?,
_ => (vec![], CompressedFormat::Lz4Legacy),
};
@@ -280,13 +285,11 @@ impl BootImagePatcher for MagiskRootPatcher {
(b"overlay.d".as_slice(), 0o750),
(b"overlay.d/sbin".as_slice(), 0o750),
] {
let mut entry = CpioEntryNew::new_directory(path);
entry.mode |= perms;
entries.push(entry);
entries.push(CpioEntry::new_directory(path, perms));
}
// Delete the original init.
entries.retain(|e| e.name != b"init");
entries.retain(|e| e.path != b"init");
// Add magiskinit.
{
@@ -294,10 +297,11 @@ impl BootImagePatcher for MagiskRootPatcher {
let mut data = vec![];
zip_entry.read_to_end(&mut data)?;
let mut entry = CpioEntryNew::new_file(b"init");
entry.mode |= 0o750;
entry.content = data;
entries.push(entry);
entries.push(CpioEntry::new_file(
b"init",
0o750,
CpioEntryData::Data(data),
));
}
// Add xz-compressed magisk32 and magisk64.
@@ -326,10 +330,12 @@ impl BootImagePatcher for MagiskRootPatcher {
stream::copy(reader, &mut writer, cancel_signal)?;
let raw_writer = writer.finish()?;
let mut entry = CpioEntryNew::new_file(target);
entry.mode |= 0o644;
entry.content = raw_writer.into_inner();
entries.push(entry);
entries.push(CpioEntry::new_file(
target,
0o644,
CpioEntryData::Data(raw_writer.into_inner()),
));
}
// Create Magisk .backup directory structure.
@@ -359,17 +365,16 @@ impl BootImagePatcher for MagiskRootPatcher {
magisk_config.push_str(&format!("RANDOMSEED={:#x}\n", self.random_seed));
}
{
// Intentially using 000 permissions to match Magisk.
let mut entry = CpioEntryNew::new_file(b".backup/.magisk");
entry.content = magisk_config.into_bytes();
entries.push(entry);
}
entries.push(CpioEntry::new_file(
b".backup/.magisk",
0,
CpioEntryData::Data(magisk_config.into_bytes()),
));
// Repack ramdisk.
cpio::sort(&mut entries);
cpio::reassign_inodes(&mut entries);
let new_ramdisk = save_ramdisk(&entries, ramdisk_format)?;
let new_ramdisk = save_ramdisk(&entries, ramdisk_format, cancel_signal)?;
match boot_image {
BootImage::V0Through2(b) => b.ramdisk = new_ramdisk,
@@ -408,7 +413,10 @@ impl OtaCertPatcher {
Self { cert }
}
pub fn get_certificates(boot_image: &BootImage) -> Result<Vec<Certificate>> {
pub fn get_certificates(
boot_image: &BootImage,
cancel_signal: &AtomicBool,
) -> Result<Vec<Certificate>> {
let mut ramdisks = vec![];
match boot_image {
@@ -420,12 +428,15 @@ impl OtaCertPatcher {
let mut certificates = vec![];
for ramdisk in ramdisks {
let (entries, _) = load_ramdisk(ramdisk)?;
let Some(entry) = entries.iter().find(|e| e.name == Self::OTACERTS_PATH) else {
let (entries, _) = load_ramdisk(ramdisk, cancel_signal)?;
let Some(entry) = entries.iter().find(|e| e.path == Self::OTACERTS_PATH) else {
continue;
};
let CpioEntryData::Data(data) = &entry.data else {
continue;
};
let mut zip = ZipArchive::new(Cursor::new(&entry.content))?;
let mut zip = ZipArchive::new(Cursor::new(&data))?;
for index in 0..zip.len() {
let zip_entry = zip.by_index(index)?;
@@ -441,16 +452,16 @@ impl OtaCertPatcher {
Ok(certificates)
}
fn patch_ramdisk(&self, data: &mut Vec<u8>) -> Result<bool> {
let (mut entries, ramdisk_format) = load_ramdisk(data)?;
let Some(entry) = entries.iter_mut().find(|e| e.name == Self::OTACERTS_PATH) else {
fn patch_ramdisk(&self, data: &mut Vec<u8>, cancel_signal: &AtomicBool) -> Result<bool> {
let (mut entries, ramdisk_format) = load_ramdisk(data, cancel_signal)?;
let Some(entry) = entries.iter_mut().find(|e| e.path == Self::OTACERTS_PATH) else {
return Ok(false);
};
// Create a new otacerts archive. The old certs are ignored since
// flashing a stock OTA will render the device unbootable.
{
let raw_writer = Cursor::new(vec![]);
let raw_writer = Cursor::new(Vec::new());
let mut writer = ZipWriter::new(raw_writer);
let options = FileOptions::default().compression_method(CompressionMethod::Stored);
writer.start_file("ota.x509.pem", options)?;
@@ -458,26 +469,26 @@ impl OtaCertPatcher {
crypto::write_pem_cert(&mut writer, &self.cert)?;
let raw_writer = writer.finish()?;
entry.content = raw_writer.into_inner();
entry.data = CpioEntryData::Data(raw_writer.into_inner());
}
// Repack ramdisk.
*data = save_ramdisk(&entries, ramdisk_format)?;
*data = save_ramdisk(&entries, ramdisk_format, cancel_signal)?;
Ok(true)
}
}
impl BootImagePatcher for OtaCertPatcher {
fn patch(&self, boot_image: &mut BootImage, _cancel_signal: &AtomicBool) -> Result<()> {
fn patch(&self, boot_image: &mut BootImage, cancel_signal: &AtomicBool) -> Result<()> {
let patched_any = match boot_image {
BootImage::V0Through2(b) => self.patch_ramdisk(&mut b.ramdisk)?,
BootImage::V3Through4(b) => self.patch_ramdisk(&mut b.ramdisk)?,
BootImage::V0Through2(b) => self.patch_ramdisk(&mut b.ramdisk, cancel_signal)?,
BootImage::V3Through4(b) => self.patch_ramdisk(&mut b.ramdisk, cancel_signal)?,
BootImage::VendorV3Through4(b) => {
let mut patched = false;
for ramdisk in &mut b.ramdisks {
if self.patch_ramdisk(ramdisk)? {
if self.patch_ramdisk(ramdisk, cancel_signal)? {
patched = true;
break;
}
avbroot/src/cli/args.rs
+1 -1
View File
@@ -45,7 +45,7 @@ pub fn main(cancel_signal: &AtomicBool) -> Result<()> {
Command::Fec(c) => fec::fec_main(&c, cancel_signal),
Command::Key(c) => key::key_main(&c),
Command::Ota(c) => ota::ota_main(&c, cancel_signal),
Command::Ramdisk(c) => ramdisk::ramdisk_main(&c),
Command::Ramdisk(c) => ramdisk::ramdisk_main(&c, cancel_signal),
// Deprecated aliases.
Command::Patch(c) => ota::patch_subcommand(&c, cancel_signal),
Command::Extract(c) => ota::extract_subcommand(&c, cancel_signal),
avbroot/src/cli/boot.rs
+10 -6
View File
@@ -13,7 +13,7 @@ use anyhow::{bail, Context, Result};
use clap::{Parser, Subcommand};
use crate::{
format::{avb::Header, bootimage::BootImage, compression::CompressedReader, cpio},
format::{avb::Header, bootimage::BootImage, compression::CompressedReader, cpio::CpioReader},
stream::{FromReader, ToWriter},
};
@@ -300,12 +300,16 @@ pub fn magisk_info_subcommand(cli: &MagiskInfoCli) -> Result<()> {
let reader = Cursor::new(ramdisk);
let reader = CompressedReader::new(reader, true)
.with_context(|| format!("Failed to load ramdisk #{i}"))?;
let entries = cpio::load(reader, false)
.with_context(|| format!("Failed to load ramdisk #{i} cpio"))?;
let mut cpio_reader = CpioReader::new(reader, false);
if let Some(e) = entries.iter().find(|e| e.name == b".backup/.magisk") {
io::stdout().write_all(&e.content)?;
return Ok(());
while let Some(entry) = cpio_reader
.next_entry()
.with_context(|| format!("Failed to read ramdisk #{i} cpio entry"))?
{
if entry.path == b".backup/.magisk" {
io::copy(&mut cpio_reader, &mut io::stdout())?;
return Ok(());
}
}
}
avbroot/src/cli/ota.rs
+1 -1
View File
@@ -1147,7 +1147,7 @@ pub fn verify_subcommand(cli: &VerifyCli, cancel_signal: &AtomicBool) -> Result<
.with_context(|| format!("Failed to read boot image: {path:?}"))?
};
let ramdisk_certs = OtaCertPatcher::get_certificates(&boot_image)
let ramdisk_certs = OtaCertPatcher::get_certificates(&boot_image, cancel_signal)
.context("Failed to read ramdisk's otacerts.zip")?;
if !ramdisk_certs.contains(&ota_cert) {
bail!("Ramdisk's otacerts.zip does not contain OTA certificate");
avbroot/src/cli/ramdisk.rs
+50 -69
View File
@@ -5,107 +5,88 @@
use std::{
fs::File,
io::{BufReader, BufWriter},
path::{Path, PathBuf},
str,
sync::atomic::AtomicBool,
};
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use crate::format::{
compression::{CompressedFormat, CompressedReader, CompressedWriter},
cpio::{self, CpioEntryNew},
use crate::{
format::{
compression::{CompressedFormat, CompressedReader, CompressedWriter},
cpio::{CpioEntryData, CpioReader, CpioWriter},
},
stream,
};
static CONTENT_BEGIN: &str = "----- BEGIN UTF-8 CONTENT -----";
static CONTENT_END: &str = "----- END UTF-8 CONTENT -----";
static CONTENT_END_NO_NEWLINE: &str = "----- END UTF-8 CONTENT (NO NEWLINE) -----";
static BINARY_BEGIN: &str = "----- BEGIN BINARY CONTENT -----";
static BINARY_END: &str = "----- END BINARY CONTENT -----";
static BINARY_END_TRUNCATED: &str = "----- END BINARY CONTENT (TRUNCATED) -----";
static NO_DATA: &str = "----- NO DATA -----";
fn print_content(data: &[u8], truncate: bool) {
if data.is_empty() {
println!("{NO_DATA}");
return;
}
if !data.contains(&b'\0') {
if let Ok(s) = str::from_utf8(data) {
if !s.contains(CONTENT_BEGIN)
&& !s.contains(CONTENT_END)
&& !s.contains(CONTENT_END_NO_NEWLINE)
{
println!("{CONTENT_BEGIN}");
print!("{s}");
if data.last() == Some(&b'\n') {
println!("{CONTENT_END}");
} else {
println!();
println!("{CONTENT_END_NO_NEWLINE}");
}
return;
}
}
}
println!("{BINARY_BEGIN}");
if data.len() > 512 && truncate {
println!("{}", data[..512].escape_ascii());
println!("{BINARY_END_TRUNCATED}");
} else {
println!("{}", data.escape_ascii());
println!("{BINARY_END}");
}
}
fn load_archive(
fn open_reader(
path: &Path,
include_trailer: bool,
) -> Result<(Vec<CpioEntryNew>, CompressedFormat)> {
) -> Result<(
CpioReader<CompressedReader<BufReader<File>>>,
CompressedFormat,
)> {
let file = File::open(path)?;
let reader = CompressedReader::new(file, true)?;
let reader = CompressedReader::new(BufReader::new(file), true)?;
let format = reader.format();
let entries = cpio::load(reader, include_trailer)?;
let cpio_reader = CpioReader::new(reader, include_trailer);
Ok((entries, format))
Ok((cpio_reader, format))
}
fn save_archive(path: &Path, entries: &[CpioEntryNew], format: CompressedFormat) -> Result<()> {
fn open_writer(
path: &Path,
format: CompressedFormat,
) -> Result<CpioWriter<CompressedWriter<BufWriter<File>>>> {
let file = File::create(path)?;
let mut writer = CompressedWriter::new(file, format)?;
cpio::save(&mut writer, entries, false)?;
writer.finish()?;
let writer = CompressedWriter::new(BufWriter::new(file), format)?;
let cpio_writer = CpioWriter::new(writer, false);
Ok(cpio_writer)
}
fn flush_writer(writer: CpioWriter<CompressedWriter<BufWriter<File>>>) -> Result<()> {
let compressed_writer = writer.finish()?;
let buf_writer = compressed_writer.finish()?;
buf_writer.into_inner()?;
Ok(())
}
pub fn ramdisk_main(cli: &RamdiskCli) -> Result<()> {
pub fn ramdisk_main(cli: &RamdiskCli, cancel_signal: &AtomicBool) -> Result<()> {
match &cli.command {
RamdiskCommand::Dump(c) => {
let (entries, format) = load_archive(&c.input, true)
let (mut reader, format) = open_reader(&c.input, true)
.with_context(|| format!("Failed to read cpio: {:?}", c.input))?;
println!("Compression format: {format:?}");
println!();
for entry in entries {
println!("{entry}");
print_content(&entry.content, !c.no_truncate);
while let Some(entry) = reader.next_entry().context("Failed to read cpio entry")? {
println!();
println!("{entry}");
}
}
RamdiskCommand::Repack(c) => {
let (entries, format) = load_archive(&c.input, false)
.with_context(|| format!("Failed to read cpio: {:?}", c.input))?;
let (mut reader, format) = open_reader(&c.input, false)
.with_context(|| format!("Failed to open cpio for reading: {:?}", c.input))?;
let mut writer = open_writer(&c.output, format)
.with_context(|| format!("Failed to open cpio for writing: {:?}", c.output))?;
save_archive(&c.output, &entries, format)
.with_context(|| format!("Failed to write cpio: {:?}", c.output))?;
while let Some(entry) = reader.next_entry().context("Failed to read cpio entry")? {
writer
.start_entry(&entry)
.context("Failed to write cpio entry")?;
if let CpioEntryData::Size(s) = &entry.data {
stream::copy_n(&mut reader, &mut writer, u64::from(*s), cancel_signal)
.context("Failed to copy cpio entry data")?;
}
}
flush_writer(writer).context("Failed to flush cpio writer")?;
}
}
avbroot/src/format/cpio.rs
+437 -143
View File
@@ -4,9 +4,10 @@
*/
use std::{
borrow::Cow,
fmt,
io::{self, Read, Write},
io::{self, Cursor, Read, Write},
ops::Range,
sync::atomic::AtomicBool,
};
use bstr::ByteSlice;
@@ -15,7 +16,9 @@ use thiserror::Error;
use crate::{
format::padding,
stream::{CountingReader, CountingWriter, FromReader, ToWriter, WriteZerosExt},
stream::{
self, CountingReader, CountingWriter, FromReader, ReadDiscardExt, ToWriter, WriteZerosExt,
},
util::NumBytes,
};
@@ -35,12 +38,9 @@ const C_ISCTG: u32 = 0o110000;
const IO_BLOCK_SIZE: u64 = 512;
/// The threshold when parsing an entry's filename where memory allocation
/// switches from allocating the exact size to resizing as necessary.
const REALLOC_NAME_THRESHOLD: usize = 1024;
/// The threshold when parsing an entry's contents where memory allocation
/// switches from allocating the exact size to resizing as necessary.
const REALLOC_DATA_THRESHOLD: usize = 1024 * 1024;
/// The threshold when reading data where memory allocation switches from
/// allocating the exact size to resizing as necessary.
const VEC_CAP_THRESHOLD: usize = 16384;
#[derive(Debug, Error)]
pub enum Error {
@@ -48,6 +48,8 @@ pub enum Error {
UnknownMagic([u8; 6]),
#[error("Hard links are not supported: {:?}", .0.as_bstr())]
HardLinksNotSupported(Vec<u8>),
#[error("Entry of type {0} should not have data: {:?}", .1.as_bstr())]
EntryHasData(CpioEntryType, Vec<u8>),
#[error("{0:?} field exceeds integer bounds")]
IntegerTooLarge(&'static str),
#[error("I/O error")]
@@ -94,56 +96,166 @@ fn write_int(mut writer: impl Write, mut value: u32) -> io::Result<()> {
}
/// Read a chunk of bytes from the reader. If `size` is less than
/// `realloc_thresh`, then the buffer is allocated with the exact size.
/// Otherwise, the buffer starts with a capacity of `realloc_thresh` and grows
/// as necessary. This avoids allocating excessive memory when the entry
/// [`VEC_CAP_THRESHOLD`], then the buffer is allocated with the exact size.
/// Otherwise, the buffer starts with a capacity of [`VEC_CAP_THRESHOLD`] and
/// grows as necessary. This avoids allocating excessive memory when the entry
/// specifies an excessively large value that's not backed by actual data.
fn read_data(mut reader: impl Read, size: usize, realloc_thresh: usize) -> io::Result<Vec<u8>> {
if size < realloc_thresh {
let mut buf = vec![0u8; size];
reader.read_exact(&mut buf)?;
Ok(buf)
} else {
let mut buf = Vec::with_capacity(realloc_thresh);
let mut offset = 0;
fn read_data(reader: impl Read, size: usize, cancel_signal: &AtomicBool) -> io::Result<Vec<u8>> {
let buf = Vec::with_capacity(size.min(VEC_CAP_THRESHOLD));
let mut cursor = Cursor::new(buf);
while offset < size {
let n = (size - offset).min(16384);
buf.resize(offset + n, 0);
reader.read_exact(&mut buf[offset..][..n])?;
offset += n;
stream::copy_n(reader, &mut cursor, size as u64, cancel_signal)?;
Ok(cursor.into_inner())
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CpioEntryType {
Pipe,
Char,
Directory,
Block,
Regular,
Symlink,
Socket,
Reserved,
Unknown(u16),
}
impl CpioEntryType {
pub fn from_mode(mode: u32) -> Self {
match mode & 0o170000 {
S_IFIFO => Self::Pipe,
S_IFCHR => Self::Char,
S_IFDIR => Self::Directory,
S_IFBLK => Self::Block,
S_IFREG => Self::Regular,
S_IFLNK => Self::Symlink,
S_IFSOCK => Self::Socket,
C_ISCTG => Self::Reserved,
m => Self::Unknown(m as u16),
}
}
Ok(buf)
pub fn to_mode(self) -> u32 {
match self {
Self::Pipe => S_IFIFO,
Self::Char => S_IFCHR,
Self::Directory => S_IFDIR,
Self::Block => S_IFBLK,
Self::Regular => S_IFREG,
Self::Symlink => S_IFLNK,
Self::Socket => S_IFSOCK,
Self::Reserved => C_ISCTG,
Self::Unknown(m) => m.into(),
}
}
}
fn file_type(mode: u32) -> u32 {
mode & 0o170000
impl fmt::Display for CpioEntryType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Pipe => write!(f, "pipe"),
Self::Char => write!(f, "character device"),
Self::Directory => write!(f, "directory"),
Self::Block => write!(f, "block device"),
Self::Regular => write!(f, "regular file"),
Self::Symlink => write!(f, "symbolic link"),
Self::Socket => write!(f, "socket"),
Self::Reserved => write!(f, "reserved"),
Self::Unknown(m) => write!(f, "unknown ({m:o})"),
}
}
}
impl Default for CpioEntryType {
fn default() -> Self {
Self::Unknown(0)
}
}
#[derive(Clone, PartialEq, Eq)]
pub enum CpioEntryData {
/// Size of entry's data. [`CpioReader`] and [`CpioWriter`] use this for
/// [`CpioEntryType::Regular`] entries to allow for lazy reads and writes.
Size(u32),
/// Entry's data. For [`CpioEntryType::Symlink`] entries, this is the
/// link target. For [`CpioEntryType::Regular`] entries, this is the file
/// content. [`CpioReader`] will never use this when reading regular files.
/// [`CpioWriter`] will write this immediately and lazy writes will not be
/// allowed.
Data(Vec<u8>),
}
impl fmt::Debug for CpioEntryData {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Size(s) => f.debug_tuple("Size").field(s).finish(),
Self::Data(d) => f.debug_tuple("Data").field(&NumBytes(d.len())).finish(),
}
}
}
impl Default for CpioEntryData {
fn default() -> Self {
Self::Size(0)
}
}
impl CpioEntryData {
pub fn size(&self) -> Result<u32> {
let size = match self {
Self::Size(s) => *s,
Self::Data(d) => d
.len()
.to_u32()
.ok_or_else(|| Error::IntegerTooLarge("data_size"))?,
};
Ok(size)
}
}
#[derive(Clone, Default, PartialEq, Eq)]
pub struct CpioEntryNew {
pub ino: u32,
pub mode: u32,
pub struct CpioEntry {
/// Inode number.
pub inode: u32,
/// File type portion of the `st_mode`-style mode.
pub file_type: CpioEntryType,
/// Permissions portion of the `st_mode`-style mode.
pub file_mode: u16,
/// Owner user ID.
pub uid: u32,
/// Owner group ID.
pub gid: u32,
/// Number of paths referencing the inode.
pub nlink: u32,
/// Modification timestamp in Unix time.
pub mtime: u32,
/// Major ID (class of device) for the device containing the inode.
pub dev_maj: u32,
/// Minor ID (specific device instance) for the device containing the inode.
pub dev_min: u32,
/// Major ID (class of device) represented by this entry. This is only
/// relevant for [`CpioEntryType::Char`] and [`CpioEntryType::Block`].
pub rdev_maj: u32,
/// Minor ID (specific device instance) represented by this entry. This is
/// only relevant for [`CpioEntryType::Char`] and [`CpioEntryType::Block`].
pub rdev_min: u32,
pub chksum: u32,
pub name: Vec<u8>,
pub content: Vec<u8>,
/// CRC32 checksum.
pub crc32: u32,
/// File path.
pub path: Vec<u8>,
/// File data.
pub data: CpioEntryData,
}
impl fmt::Debug for CpioEntryNew {
impl fmt::Debug for CpioEntry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("CpioEntryNew")
.field("ino", &self.ino)
.field("mode", &self.mode)
f.debug_struct("CpioEntry")
.field("inode", &self.inode)
.field("file_type", &self.file_type)
.field("file_mode", &self.file_mode)
.field("uid", &self.uid)
.field("gid", &self.gid)
.field("nlink", &self.nlink)
@@ -152,84 +264,87 @@ impl fmt::Debug for CpioEntryNew {
.field("dev_min", &self.dev_min)
.field("rdev_maj", &self.rdev_maj)
.field("rdev_min", &self.rdev_min)
.field("chksum", &self.chksum)
.field("name", &self.name.as_bstr())
.field("content", &NumBytes(self.content.len()))
.field("crc32", &self.crc32)
.field("path", &self.path.as_bstr())
.field("data", &self.data)
.finish()
}
}
impl fmt::Display for CpioEntryNew {
impl fmt::Display for CpioEntry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let file_type_str = match file_type(self.mode) {
S_IFIFO => Cow::Borrowed("pipe"),
S_IFCHR => Cow::Borrowed("character device"),
S_IFDIR => Cow::Borrowed("directory"),
S_IFBLK => Cow::Borrowed("block device"),
S_IFREG => Cow::Borrowed("regular file"),
S_IFLNK => Cow::Borrowed("symbolic link"),
S_IFSOCK => Cow::Borrowed("socket"),
C_ISCTG => Cow::Borrowed("reserved"),
m => Cow::Owned(format!("unknown ({m:o})")),
};
writeln!(f, "Filename: {:?}", self.name.as_bstr())?;
writeln!(f, "Filetype: {file_type_str}")?;
writeln!(f, "Inode: {}", self.ino)?;
writeln!(f, "Mode: {:o}", self.mode)?;
writeln!(f, "UID: {}", self.uid)?;
writeln!(f, "GID: {}", self.gid)?;
writeln!(f, "Links: {}", self.nlink)?;
writeln!(f, "Modified: {}", self.mtime)?;
writeln!(f, "Device: {:x},{:x}", self.dev_maj, self.dev_min)?;
writeln!(f, "Device ID: {:x},{:x}", self.rdev_maj, self.rdev_min)?;
writeln!(f, "Checksum: {:x}", self.chksum)?;
writeln!(f, "Content: {:?}", NumBytes(self.content.len()))?;
writeln!(f, "Path: {:?}", self.path.as_bstr())?;
match &self.data {
CpioEntryData::Size(s) => {
writeln!(f, "Data: {:?}", &NumBytes(*s))?;
}
CpioEntryData::Data(d) => {
if self.file_type == CpioEntryType::Symlink {
writeln!(f, "Data: {:?}", d.as_bstr())?;
} else {
writeln!(f, "Data: {:?}", &NumBytes(d.len()))?;
}
}
}
writeln!(f, "Inode: {}", self.inode)?;
writeln!(f, "Type: {}", self.file_type)?;
writeln!(f, "Mode: {:o}", self.file_mode)?;
writeln!(f, "UID: {}", self.uid)?;
writeln!(f, "GID: {}", self.gid)?;
writeln!(f, "Links: {}", self.nlink)?;
writeln!(f, "Modtime: {}", self.mtime)?;
writeln!(f, "Idevice: {:x},{:x}", self.dev_maj, self.dev_min)?;
writeln!(f, "Rdevice: {:x},{:x}", self.rdev_maj, self.rdev_min)?;
write!(f, "CRC32: {:x}", self.crc32)?;
Ok(())
}
}
impl CpioEntryNew {
impl CpioEntry {
pub fn new_trailer() -> Self {
Self {
// Must be 1 for CRC format.
nlink: 1,
name: CPIO_TRAILER.to_vec(),
path: CPIO_TRAILER.to_vec(),
..Default::default()
}
}
pub fn new_symlink(link_target: &[u8], name: &[u8]) -> Self {
pub fn new_symlink(path: &[u8], link_target: &[u8]) -> Self {
Self {
mode: S_IFLNK | 0o777,
file_type: CpioEntryType::Symlink,
file_mode: 0o777,
nlink: 1,
name: name.to_owned(),
content: link_target.to_owned(),
path: path.to_owned(),
data: CpioEntryData::Data(link_target.to_owned()),
..Default::default()
}
}
pub fn new_directory(name: &[u8]) -> Self {
pub fn new_directory(path: &[u8], mode: u16) -> Self {
Self {
mode: S_IFDIR,
file_type: CpioEntryType::Directory,
file_mode: mode,
nlink: 1,
name: name.to_owned(),
path: path.to_owned(),
..Default::default()
}
}
pub fn new_file(name: &[u8]) -> Self {
pub fn new_file(path: &[u8], mode: u16, data: CpioEntryData) -> Self {
Self {
mode: S_IFREG,
file_type: CpioEntryType::Regular,
file_mode: mode,
nlink: 1,
name: name.to_owned(),
path: path.to_owned(),
data,
..Default::default()
}
}
}
impl<R: Read> FromReader<R> for CpioEntryNew {
impl<R: Read> FromReader<R> for CpioEntry {
type Error = Error;
fn from_reader(reader: R) -> Result<Self> {
@@ -242,45 +357,59 @@ impl<R: Read> FromReader<R> for CpioEntryNew {
return Err(Error::UnknownMagic(magic));
}
let ino = read_int(&mut reader)?;
let inode = read_int(&mut reader)?;
let mode = read_int(&mut reader)?;
let uid = read_int(&mut reader)?;
let gid = read_int(&mut reader)?;
let nlink = read_int(&mut reader)?;
let mtime = read_int(&mut reader)?;
let filesize = read_int(&mut reader)?;
let file_size = read_int(&mut reader)?;
let dev_maj = read_int(&mut reader)?;
let dev_min = read_int(&mut reader)?;
let rdev_maj = read_int(&mut reader)?;
let rdev_min = read_int(&mut reader)?;
let namesize = read_int(&mut reader)?;
let chksum = read_int(&mut reader)?;
let path_size = read_int(&mut reader)?;
let crc32 = read_int(&mut reader)?;
let mut name = read_data(
let mut path = read_data(
&mut reader,
namesize.to_usize().unwrap(),
REALLOC_NAME_THRESHOLD,
path_size.to_usize().unwrap(),
&AtomicBool::new(false),
)?;
if name.last() != Some(&b'\0') {
if path.last() != Some(&b'\0') {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"Filename is not NULL-terminated",
)
.into());
}
name.pop();
path.pop();
padding::read_discard(&mut reader, 4)?;
let content = read_data(
&mut reader,
filesize.to_usize().unwrap(),
REALLOC_DATA_THRESHOLD,
)?;
padding::read_discard(&mut reader, 4)?;
let file_type = CpioEntryType::from_mode(mode);
let data = match file_type {
// Handled by CpioReader for streaming reads.
CpioEntryType::Regular => CpioEntryData::Size(file_size),
// Symlinks store the target in the file data.
CpioEntryType::Symlink => {
let content = read_data(
&mut reader,
file_size.to_usize().unwrap(),
&AtomicBool::new(false),
)?;
padding::read_discard(&mut reader, 4)?;
CpioEntryData::Data(content)
}
// No other entry type should have data.
t if file_size != 0 => return Err(Error::EntryHasData(t, path)),
_ => CpioEntryData::Size(0),
};
Ok(Self {
ino,
mode,
inode,
file_type,
file_mode: (mode & 0o7777) as u16,
uid,
gid,
nlink,
@@ -289,76 +418,241 @@ impl<R: Read> FromReader<R> for CpioEntryNew {
dev_min,
rdev_maj,
rdev_min,
chksum,
name,
content,
crc32,
path,
data,
})
}
}
impl<W: Write> ToWriter<W> for CpioEntryNew {
impl<W: Write> ToWriter<W> for CpioEntry {
type Error = Error;
fn to_writer(&self, writer: W) -> Result<()> {
let mut writer = CountingWriter::new(writer);
let filesize = self
.content
.len()
.to_u32()
.ok_or_else(|| Error::IntegerTooLarge("filesize"))?;
let namesize = self
.name
let path_size = self
.path
.len()
.checked_add(1)
.and_then(|s| s.to_u32())
.ok_or_else(|| Error::IntegerTooLarge("filesize"))?;
.ok_or_else(|| Error::IntegerTooLarge("path_size"))?;
if self.chksum == 0 {
let file_size = self.data.size()?;
if file_size != 0
&& self.file_type != CpioEntryType::Regular
&& self.file_type != CpioEntryType::Symlink
{
return Err(Error::EntryHasData(self.file_type, self.path.clone()));
}
if self.crc32 == 0 {
writer.write_all(MAGIC_NEW)?;
} else {
writer.write_all(MAGIC_NEW_CRC)?;
}
write_int(&mut writer, self.ino)?;
write_int(&mut writer, self.mode)?;
let mode = self.file_type.to_mode() | u32::from(self.file_mode & 0o7777);
write_int(&mut writer, self.inode)?;
write_int(&mut writer, mode)?;
write_int(&mut writer, self.uid)?;
write_int(&mut writer, self.gid)?;
write_int(&mut writer, self.nlink)?;
write_int(&mut writer, self.mtime)?;
write_int(&mut writer, filesize)?;
write_int(&mut writer, file_size)?;
write_int(&mut writer, self.dev_maj)?;
write_int(&mut writer, self.dev_min)?;
write_int(&mut writer, self.rdev_maj)?;
write_int(&mut writer, self.rdev_min)?;
write_int(&mut writer, namesize)?;
write_int(&mut writer, self.chksum)?;
write_int(&mut writer, path_size)?;
write_int(&mut writer, self.crc32)?;
writer.write_all(&self.name)?;
writer.write_all(&self.path)?;
writer.write_zeros_exact(1)?;
padding::write_zeros(&mut writer, 4)?;
writer.write_all(&self.content)?;
padding::write_zeros(&mut writer, 4)?;
if let CpioEntryData::Data(d) = &self.data {
writer.write_all(d)?;
padding::write_zeros(&mut writer, 4)?;
}
Ok(())
}
}
pub fn load(mut reader: impl Read, include_trailer: bool) -> Result<Vec<CpioEntryNew>> {
let mut entries = vec![];
pub struct CpioReader<R: Read> {
reader: R,
include_trailer: bool,
range: Option<Range<u64>>,
done: bool,
}
loop {
let entry = CpioEntryNew::from_reader(&mut reader)?;
if file_type(entry.mode) != S_IFDIR && entry.nlink > 1 {
return Err(Error::HardLinksNotSupported(entry.name));
impl<R: Read> CpioReader<R> {
pub fn new(reader: R, include_trailer: bool) -> Self {
Self {
reader,
include_trailer,
range: None,
done: false,
}
}
pub fn into_inner(self) -> R {
self.reader
}
fn skip_data(&mut self) -> io::Result<()> {
if let Some(range) = &mut self.range {
// This cannot overflow because cpio file sizes are 32 bit.
let n = range.end - range.start + padding::calc(range.end, 4);
self.reader.read_discard_exact(n)?;
self.range = None;
}
if entry.name == CPIO_TRAILER {
if include_trailer {
entries.push(entry);
Ok(())
}
pub fn next_entry(&mut self) -> Result<Option<CpioEntry>> {
if self.done {
return Ok(None);
}
self.skip_data()?;
let entry = CpioEntry::from_reader(&mut self.reader)?;
if entry.path == CPIO_TRAILER {
self.done = true;
if !self.include_trailer {
return Ok(None);
}
break;
} else if let CpioEntryData::Size(s) = entry.data {
self.range = Some(0..u64::from(s));
}
Ok(Some(entry))
}
}
impl<R: Read> Read for CpioReader<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let Some(range) = &mut self.range else {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"No entry opened",
));
};
let to_read = (range.end - range.start).min(buf.len() as u64) as usize;
let n = self.reader.read(&mut buf[..to_read])?;
range.start += n as u64;
Ok(n)
}
}
pub struct CpioWriter<W: Write> {
writer: CountingWriter<W>,
pad_to_block_size: bool,
range: Option<Range<u64>>,
max_inode: u32,
}
impl<W: Write> CpioWriter<W> {
pub fn new(writer: W, pad_to_block_size: bool) -> Self {
Self {
writer: CountingWriter::new(writer),
pad_to_block_size,
range: None,
max_inode: 0,
}
}
fn finish_entry(&mut self) -> io::Result<()> {
if let Some(range) = &mut self.range {
// This cannot overflow because cpio file sizes are 32 bit.
let n = range.end - range.start + padding::calc(range.end, 4);
self.writer.write_zeros_exact(n)?;
self.range = None;
}
Ok(())
}
pub fn start_entry(&mut self, entry: &CpioEntry) -> Result<()> {
self.finish_entry()?;
entry.to_writer(&mut self.writer)?;
if let CpioEntryData::Size(s) = entry.data {
self.range = Some(0..u64::from(s));
}
self.max_inode = self.max_inode.max(entry.inode);
Ok(())
}
pub fn finish(mut self) -> Result<W> {
self.finish_entry()?;
let mut trailer = CpioEntry::new_trailer();
trailer.inode = self.max_inode.wrapping_add(1);
self.start_entry(&trailer)?;
// Pad until the end of the block.
if self.pad_to_block_size {
padding::write_zeros(&mut self.writer, IO_BLOCK_SIZE)?;
}
Ok(self.writer.finish().0)
}
}
impl<W: Write> Write for CpioWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let Some(range) = &mut self.range else {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"No entry started",
));
};
let to_write = (range.end - range.start).min(buf.len() as u64) as usize;
let n = self.writer.write(&buf[..to_write])?;
range.start += n as u64;
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.writer.flush()
}
}
pub fn load(
reader: impl Read,
include_trailer: bool,
cancel_signal: &AtomicBool,
) -> Result<Vec<CpioEntry>> {
let mut cpio_reader = CpioReader::new(reader, include_trailer);
let mut entries = vec![];
while let Some(mut entry) = cpio_reader.next_entry()? {
stream::check_cancel(cancel_signal)?;
if entry.file_type != CpioEntryType::Directory && entry.nlink > 1 {
return Err(Error::HardLinksNotSupported(entry.path.clone()));
}
if let CpioEntryData::Size(s) = entry.data {
let data = read_data(&mut cpio_reader, s.to_usize().unwrap(), cancel_signal)?;
entry.data = CpioEntryData::Data(data);
}
entries.push(entry);
@@ -367,35 +661,35 @@ pub fn load(mut reader: impl Read, include_trailer: bool) -> Result<Vec<CpioEntr
Ok(entries)
}
pub fn sort(entries: &mut [CpioEntryNew]) {
entries.sort_by(|a, b| a.name.cmp(&b.name));
pub fn sort(entries: &mut [CpioEntry]) {
entries.sort_by(|a, b| a.path.cmp(&b.path));
}
pub fn reassign_inodes(entries: &mut [CpioEntryNew]) {
pub fn reassign_inodes(entries: &mut [CpioEntry]) {
let mut inode = 300000;
for entry in entries {
entry.ino = inode;
entry.inode = inode;
inode += 1;
}
}
pub fn save(writer: impl Write, entries: &[CpioEntryNew], pad_to_block_size: bool) -> Result<()> {
let mut writer = CountingWriter::new(writer);
pub fn save(
writer: impl Write,
entries: &[CpioEntry],
pad_to_block_size: bool,
cancel_signal: &AtomicBool,
) -> Result<()> {
let mut cpio_writer = CpioWriter::new(writer, pad_to_block_size);
for entry in entries {
entry.to_writer(&mut writer)?;
stream::check_cancel(cancel_signal)?;
cpio_writer.start_entry(entry)?;
// CpioEntryData::Data will have already been written.
}
let mut trailer = CpioEntryNew::new_trailer();
// 1 higher than the highest inode if possible.
trailer.ino = entries.iter().map(|e| e.ino).max().map_or(0, |i| i + 1);
trailer.to_writer(&mut writer)?;
// Pad until the end of the block.
if pad_to_block_size {
padding::write_zeros(&mut writer, IO_BLOCK_SIZE)?;
}
cpio_writer.finish()?;
Ok(())
}
avbroot/src/util.rs
+4 -3
View File
@@ -5,17 +5,18 @@
use std::{fmt, path::Path};
use num_traits::PrimInt;
use quick_protobuf::{BytesReader, MessageRead, MessageWrite, Writer};
pub const ZEROS: [u8; 16384] = [0u8; 16384];
/// A small wrapper to format a number as a size in bytes.
#[derive(Clone, Copy)]
pub struct NumBytes(pub usize);
pub struct NumBytes<T: PrimInt>(pub T);
impl fmt::Debug for NumBytes {
impl<T: PrimInt + fmt::Debug> fmt::Debug for NumBytes<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.0 == 1 {
if self.0 == T::one() {
write!(f, "<{:?} byte>", self.0)
} else {
write!(f, "<{:?} bytes>", self.0)
avbroot/tests/cpio.rs
+50
View File
@@ -0,0 +1,50 @@
/*
* SPDX-FileCopyrightText: 2023 Andrew Gunnerson
* SPDX-License-Identifier: GPL-3.0-only
*/
use std::io::{self, Cursor};
use avbroot::{
self,
format::cpio::{CpioEntryType, CpioReader, CpioWriter},
util,
};
#[test]
fn round_trip_archive() {
let data = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/data/archive.cpio",
));
assert_ne!(data.len() % 512, 0);
for pad_to_block_size in [false, true] {
println!("Pad to block size: {pad_to_block_size}");
let reader = Cursor::new(data);
let mut cpio_reader = CpioReader::new(reader, false);
let writer = Cursor::new(Vec::new());
let mut cpio_writer = CpioWriter::new(writer, pad_to_block_size);
while let Some(entry) = cpio_reader.next_entry().unwrap() {
cpio_writer.start_entry(&entry).unwrap();
if entry.file_type == CpioEntryType::Regular {
io::copy(&mut cpio_reader, &mut cpio_writer).unwrap();
}
}
let writer = cpio_writer.finish().unwrap();
let new_data = writer.get_ref().as_slice();
if pad_to_block_size {
assert!(new_data.starts_with(data));
assert!(util::is_zero(&new_data[data.len()..]));
assert_eq!(new_data.len() % 512, 0);
} else {
assert_eq!(new_data, data);
}
}
}
avbroot/tests/data/archive.cpio
BIN
View File
Binary file not shown.
fuzz/src/bin/cpio.rs
+3 -2
View File
@@ -1,6 +1,6 @@
#[cfg(not(windows))]
mod fuzz {
use std::io::Cursor;
use std::{io::Cursor, sync::atomic::AtomicBool};
use avbroot::format::cpio;
use honggfuzz::fuzz;
@@ -8,8 +8,9 @@ mod fuzz {
pub fn main() {
loop {
fuzz!(|data: &[u8]| {
let cancel_signal = AtomicBool::new(false);
let reader = Cursor::new(data);
let _ = cpio::load(reader, true);
let _ = cpio::load(reader, true, &cancel_signal);
});
}
}