From 9d35c5772d5bacf6bc79c8d63b732e2f14ae48df Mon Sep 17 00:00:00 2001 From: trivernis Date: Sat, 26 Sep 2020 11:03:58 +0200 Subject: [PATCH] Add scan commmand that also fixes lengths The scan command looks for different section counts in the locations table and optionally fixes it. Signed-off-by: trivernis --- Cargo.lock | 211 +++++++++++++++++++++++++++++++++----------- Cargo.toml | 6 +- README.md | 1 + src/lib.rs | 1 + src/main.rs | 11 +++ src/region_file.rs | 111 ++++++++++++++++++++--- src/scan.rs | 38 ++++++++ src/world_folder.rs | 65 +++++++++++--- 8 files changed, 365 insertions(+), 79 deletions(-) create mode 100644 src/scan.rs diff --git a/Cargo.lock b/Cargo.lock index 82df4b4..84733a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + [[package]] name = "bitflags" version = "1.2.1" @@ -54,14 +60,81 @@ dependencies = [ ] [[package]] -name = "cloudabi" -version = "0.1.0" +name = "console" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" +checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" dependencies = [ - "bitflags", + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "termios", + "unicode-width", + "winapi", + "winapi-util", +] + +[[package]] +name = "crossbeam-channel" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" +dependencies = [ + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "maybe-uninit", ] +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "heck" version = "0.3.1" @@ -81,12 +154,15 @@ dependencies = [ ] [[package]] -name = "instant" -version = "0.1.7" +name = "indicatif" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" +checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" dependencies = [ - "cfg-if", + "console", + "lazy_static", + "number_prefix", + "regex", ] [[package]] @@ -102,21 +178,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" [[package]] -name = "lock_api" -version = "0.4.1" +name = "maybe-uninit" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" dependencies = [ - "scopeguard", + "autocfg", ] [[package]] name = "minecraft-regions-tool" -version = "0.1.0" +version = "0.2.0" dependencies = [ "byteorder", - "num_cpus", - "scheduled-thread-pool", + "indicatif", + "rayon", "structopt", ] @@ -131,30 +213,10 @@ dependencies = [ ] [[package]] -name = "parking_lot" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.0" +name = "number_prefix" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" -dependencies = [ - "cfg-if", - "cloudabi", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", -] +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" [[package]] name = "proc-macro-error" @@ -199,31 +261,50 @@ dependencies = [ ] [[package]] -name = "redox_syscall" -version = "0.1.57" +name = "rayon" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +checksum = "cfd016f0c045ad38b5251be2c9c0ab806917f82da4d36b2a327e5166adad9270" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] [[package]] -name = "scheduled-thread-pool" -version = "0.2.5" +name = "rayon-core" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +checksum = "e8c4fec834fb6e6d2dd5eece3c7b432a52f0ba887cf40e595190c4107edc08bf" dependencies = [ - "parking_lot", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", ] [[package]] -name = "scopeguard" -version = "1.1.0" +name = "regex" +version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "regex-syntax", +] [[package]] -name = "smallvec" -version = "1.4.2" +name = "regex-syntax" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "strsim" @@ -266,6 +347,25 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "terminal_size" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "termios" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" +dependencies = [ + "libc", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -321,6 +421,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 7c8bb11..17fb90a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "minecraft-regions-tool" -version = "0.1.0" +version = "0.2.0" authors = ["trivernis "] edition = "2018" license = "GPL-3.0" @@ -13,5 +13,5 @@ repository = "https://github.com/Trivernis/minecraft-regions-tool" [dependencies] byteorder = "1.3.4" structopt = "0.3.18" -scheduled-thread-pool = "0.2.5" -num_cpus = "1.13.0" \ No newline at end of file +rayon = "1.4.0" +indicatif = "0.15.0" \ No newline at end of file diff --git a/README.md b/README.md index a3988b9..0240d31 100644 --- a/README.md +++ b/README.md @@ -17,4 +17,5 @@ ARGS: SUBCOMMANDS: count Return the total number of chunks in the world help Prints this message or the help of the given subcommand(s) + scan Scan for errors in the region files and optionally fix them ``` diff --git a/src/lib.rs b/src/lib.rs index 10883dc..f3b4e59 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,3 @@ pub mod region_file; +pub mod scan; pub mod world_folder; diff --git a/src/main.rs b/src/main.rs index 8d3af3d..646dbbd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,16 @@ struct Opt { enum SubCommand { /// Return the total number of chunks in the world Count, + + /// Scan for errors in the region files and optionally fix them + Scan(ScanOptions), +} + +#[derive(StructOpt, Debug)] +#[structopt()] +struct ScanOptions { + #[structopt(short, long)] + fix: bool, } fn main() { @@ -24,5 +34,6 @@ fn main() { let world = WorldFolder::new(opt.input); match opt.sub_command { SubCommand::Count => println!("Chunk Count: {}", world.count_chunks().unwrap()), + SubCommand::Scan(opt) => world.scan_files(opt.fix).unwrap(), } } diff --git a/src/region_file.rs b/src/region_file.rs index e473668..76fb1cd 100644 --- a/src/region_file.rs +++ b/src/region_file.rs @@ -1,16 +1,19 @@ -use byteorder::{BigEndian, ByteOrder}; -use std::io::{Read, Result}; +use crate::scan::ScanStatistics; +use byteorder::{BigEndian, ByteOrder, ReadBytesExt}; +use std::fs::File; +use std::io::{BufReader, BufWriter, Read, Result, Seek, SeekFrom, Write}; const BLOCK_SIZE: usize = 4096; pub struct RegionFile { - reader: Box, + reader: BufReader, locations: Locations, + #[allow(dead_code)] timestamps: Timestamps, } impl RegionFile { - pub fn new(reader: Box) -> Result { + pub fn new(reader: BufReader) -> Result { let mut locations_raw = [0u8; BLOCK_SIZE]; let mut timestamps_raw = [0u8; BLOCK_SIZE]; let mut reader = reader; @@ -24,20 +27,67 @@ impl RegionFile { }) } + /// Writes a corrected version of the region file back to the disk + pub fn write(&self, writer: &mut BufWriter) -> Result<()> { + let location_bytes = self.locations.to_bytes(); + writer.write_all(&location_bytes.as_slice())?; + + writer.flush() + } + /// Returns the number of chunks in the file pub fn count_chunks(&self) -> usize { - let mut count = 0; - for x in 0..32 { - for z in 0..32 { - if !(self.locations.get_chunk_offset(x, z) == Some(0) - && self.locations.get_chunk_sectors(x, z) == Some(0)) - { - count += 1; + return self.locations.valid_entries().len(); + } + + /// Scans the chunk entries for possible errors + pub fn scan_chunks(&mut self) -> Result { + let mut statistic = ScanStatistics::new(); + + let entries = self.locations.valid_entries(); + let mut corrected_entries = Vec::new(); + statistic.total_chunks = entries.len() as u64; + + for (offset, sections) in &entries { + self.reader + .seek(SeekFrom::Start(*offset as u64 * BLOCK_SIZE as u64))?; + match self.read_chunk() { + Ok(chunk) => { + let chunk_sections = ((chunk.length + 4) as f64 / BLOCK_SIZE as f64).ceil(); + if *sections != chunk_sections as u8 || chunk.length >= 1_048_576 { + statistic.invalid_length += 1; + corrected_entries.push((*offset, chunk_sections as u8)); + } else { + corrected_entries.push((*offset, *sections)); + } + } + Err(e) => { + println!("Failed to read chunk at {}: {}", offset, e); } } } + self.locations.set_entries(corrected_entries); - return count; + Ok(statistic) + } + + /// Reads a chunk at the current location + fn read_chunk(&mut self) -> Result { + let mut length_raw = [0u8; 4]; + self.reader.read_exact(&mut length_raw)?; + let length = BigEndian::read_u32(&length_raw); + let compression_type = self.reader.read_u8()?; + + if length > 0 { + self.reader.seek(SeekFrom::Current((length - 1) as i64))?; + } else { + self.reader.seek(SeekFrom::Current((length) as i64))?; + } + + Ok(Chunk { + length, + compression_type, + }) } } @@ -51,8 +101,8 @@ impl Locations { let mut locations = Vec::new(); for i in (0..BLOCK_SIZE - 1).step_by(4) { - let mut offset = BigEndian::read_u32(&bytes[i..i + 4]); - offset = offset >> 1; + let offset_raw = [0u8, bytes[i], bytes[i + 1], bytes[i + 2]]; + let offset = BigEndian::read_u32(&offset_raw); let count = bytes[i + 3]; locations.push((offset, count)); } @@ -60,6 +110,20 @@ impl Locations { Self { inner: locations } } + /// Returns the byte representation of the locations table + pub fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + + for (offset, sections) in &self.inner { + let mut offset_raw = [0u8; 4]; + BigEndian::write_u32(&mut offset_raw, *offset); + bytes.append(&mut offset_raw[1..4].to_vec()); + bytes.push(*sections); + } + + bytes + } + /// Returns the offset of a chunk pub fn get_chunk_offset(&self, x: usize, z: usize) -> Option { let index = x % 32 + (z % 32) * 32; @@ -71,6 +135,19 @@ impl Locations { let index = x % 32 + (z % 32) * 32; self.inner.get(index).map(|e| (*e).1) } + + /// Returns chunk entry list + pub fn valid_entries(&self) -> Vec<(u32, u8)> { + self.inner + .iter() + .filter_map(|e| if (*e).0 >= 2 { Some(*e) } else { None }) + .collect() + } + + /// Replaces the entry list with a new one + pub fn set_entries(&mut self, entries: Vec<(u32, u8)>) { + self.inner = entries; + } } #[derive(Debug)] @@ -89,3 +166,9 @@ impl Timestamps { Self { inner: timestamps } } } + +#[derive(Debug)] +pub struct Chunk { + pub length: u32, + pub compression_type: u8, +} diff --git a/src/scan.rs b/src/scan.rs new file mode 100644 index 0000000..0edc919 --- /dev/null +++ b/src/scan.rs @@ -0,0 +1,38 @@ +use std::fmt::{Display, Formatter, Result}; +use std::ops::Add; + +#[derive(Clone, Debug)] +pub struct ScanStatistics { + pub total_chunks: u64, + pub invalid_length: u64, +} + +impl ScanStatistics { + pub fn new() -> Self { + Self { + invalid_length: 0, + total_chunks: 0, + } + } +} + +impl Add for ScanStatistics { + type Output = Self; + + fn add(mut self, rhs: Self) -> Self::Output { + self.invalid_length += rhs.invalid_length; + self.total_chunks += rhs.total_chunks; + + self + } +} + +impl Display for ScanStatistics { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + write!( + f, + "Total Chunks: {}\nChunks with invalid length: {}", + self.total_chunks, self.invalid_length + ) + } +} diff --git a/src/world_folder.rs b/src/world_folder.rs index bdbf964..b4c96a3 100644 --- a/src/world_folder.rs +++ b/src/world_folder.rs @@ -1,10 +1,14 @@ use crate::region_file::RegionFile; -use std::ffi::OsStr; +use crate::scan::ScanStatistics; +use indicatif::{ProgressBar, ProgressStyle}; +use rayon::prelude::*; use std::fs; -use std::fs::File; +use std::fs::{File, OpenOptions}; use std::io; -use std::io::BufReader; +use std::io::{BufReader, BufWriter}; +use std::ops::Add; use std::path::PathBuf; +use std::sync::{Arc, Mutex}; pub struct WorldFolder { path: PathBuf, @@ -15,19 +19,58 @@ impl WorldFolder { Self { path } } + /// Counts all chunks of a world pub fn count_chunks(&self) -> io::Result { let mut count = 0u64; - let region_file_path = self.path.join(PathBuf::from("region")); - for file in fs::read_dir(region_file_path)? { - let file_path = file?.path(); - if file_path.extension() == Some(OsStr::new("mca")) { - let f = File::open(file_path)?; - let region_file = RegionFile::new(Box::new(BufReader::new(f)))?; - count += region_file.count_chunks() as u64; - } + for file in self.region_file_paths() { + let f = File::open(file)?; + let region_file = RegionFile::new(BufReader::new(f))?; + count += region_file.count_chunks() as u64; } Ok(count) } + + pub fn scan_files(&self, fix: bool) -> io::Result<()> { + let paths = self.region_file_paths(); + let bar = Arc::new(Mutex::new(ProgressBar::new(paths.len() as u64))); + bar.lock().unwrap().set_style( + ProgressStyle::default_bar().template("[{eta_precise}] {wide_bar} {pos}/{len} "), + ); + + let statistic: ScanStatistics = paths + .par_iter() + .filter_map(|file| { + let f = OpenOptions::new().read(true).open(file).ok()?; + let mut region_file = RegionFile::new(BufReader::new(f)).ok()?; + + let result = region_file.scan_chunks().ok()?; + if fix { + let f = OpenOptions::new().write(true).open(file).ok()?; + let mut writer = BufWriter::new(f); + region_file.write(&mut writer).ok()?; + } + bar.lock().unwrap().inc(1); + + Some(result) + }) + .reduce(|| ScanStatistics::new(), |a, b| a.add(b)); + + bar.lock().unwrap().finish_and_clear(); + + println!("{}", statistic); + + Ok(()) + } + + /// Returns a list of region file paths for the world folder + fn region_file_paths(&self) -> Vec { + let region_file_path = self.path.join(PathBuf::from("region")); + + fs::read_dir(region_file_path) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .collect() + } }