From 2addee73485ce5cb3c1943269be03030d6258c6b Mon Sep 17 00:00:00 2001 From: trivernis Date: Sat, 26 Sep 2020 14:21:52 +0200 Subject: [PATCH] Add fixing of compression method Signed-off-by: trivernis --- Cargo.lock | 89 ++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 7 +++- README.md | 2 +- src/main.rs | 52 +++++++++++++++++++++++++- src/region_file.rs | 48 ++++++++++++++++-------- src/scan.rs | 9 +++-- src/world_folder.rs | 49 +++++++++++++------------ 7 files changed, 207 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84733a9..95d3b3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,14 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -59,6 +68,17 @@ dependencies = [ "vec_map", ] +[[package]] +name = "colored" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" +dependencies = [ + "atty", + "lazy_static", + "winapi", +] + [[package]] name = "console" version = "0.12.0" @@ -135,6 +155,19 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "heck" version = "0.3.1" @@ -153,6 +186,15 @@ dependencies = [ "libc", ] +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + [[package]] name = "indicatif" version = "0.15.0" @@ -177,12 +219,27 @@ version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if", +] + [[package]] name = "maybe-uninit" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + [[package]] name = "memoffset" version = "0.5.6" @@ -194,10 +251,13 @@ dependencies = [ [[package]] name = "minecraft-regions-tool" -version = "0.2.0" +version = "0.3.0" dependencies = [ "byteorder", + "colored", + "env_logger", "indicatif", + "log", "rayon", "structopt", ] @@ -251,6 +311,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.7" @@ -291,7 +357,10 @@ version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", + "thread_local", ] [[package]] @@ -347,6 +416,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.1.13" @@ -375,6 +453,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + [[package]] name = "unicode-segmentation" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 17fb90a..1ed1451 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "minecraft-regions-tool" -version = "0.2.0" +version = "0.3.0" authors = ["trivernis "] edition = "2018" license = "GPL-3.0" @@ -14,4 +14,7 @@ repository = "https://github.com/Trivernis/minecraft-regions-tool" byteorder = "1.3.4" structopt = "0.3.18" rayon = "1.4.0" -indicatif = "0.15.0" \ No newline at end of file +indicatif = "0.15.0" +log = "0.4.11" +env_logger ="0.7.1" +colored = "2.0.0" \ No newline at end of file diff --git a/README.md b/README.md index 0240d31..818861b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A utility to perform operations on minecraft region files ## Usage -```sh +``` USAGE: minecraft-regions-tool diff --git a/src/main.rs b/src/main.rs index 646dbbd..a061786 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,6 @@ +use colored::*; +use env_logger::Env; +use log::Level; use minecraft_regions_tool::world_folder::WorldFolder; use std::path::PathBuf; use structopt::StructOpt; @@ -8,6 +11,10 @@ struct Opt { #[structopt(parse(from_os_str))] input: PathBuf, + /// Forces verbose output + #[structopt(short, long)] + verbose: bool, + #[structopt(subcommand)] sub_command: SubCommand, } @@ -31,9 +38,50 @@ struct ScanOptions { fn main() { let opt: Opt = Opt::from_args(); + build_logger(opt.verbose); let world = WorldFolder::new(opt.input); match opt.sub_command { - SubCommand::Count => println!("Chunk Count: {}", world.count_chunks().unwrap()), - SubCommand::Scan(opt) => world.scan_files(opt.fix).unwrap(), + SubCommand::Count => log::info!("Chunk Count: {}", world.count_chunks().unwrap()), + SubCommand::Scan(opt) => { + if opt.fix { + log::info!("Fixing fixable errors."); + } + log::info!("Scanning Region files for errors..."); + log::info!("Scan Results:\n{}", world.scan_files(opt.fix).unwrap()) + } + } +} + +fn build_logger(verbose: bool) { + env_logger::Builder::from_env(Env::default().default_filter_or(if verbose { + "debug" + } else { + "info" + })) + .format(|buf, record| { + use std::io::Write; + let color = get_level_style(record.level()); + writeln!( + buf, + "{}: {}", + record + .level() + .to_string() + .to_lowercase() + .as_str() + .color(color), + record.args() + ) + }) + .init(); +} + +fn get_level_style(level: Level) -> colored::Color { + match level { + Level::Trace => colored::Color::Magenta, + Level::Debug => colored::Color::Blue, + Level::Info => colored::Color::Green, + Level::Warn => colored::Color::Yellow, + Level::Error => colored::Color::Red, } } diff --git a/src/region_file.rs b/src/region_file.rs index 76fb1cd..3104066 100644 --- a/src/region_file.rs +++ b/src/region_file.rs @@ -1,22 +1,28 @@ use crate::scan::ScanStatistics; -use byteorder::{BigEndian, ByteOrder, ReadBytesExt}; -use std::fs::File; +use byteorder::{BigEndian, ByteOrder, ReadBytesExt, WriteBytesExt}; +use std::fs::{File, OpenOptions}; use std::io::{BufReader, BufWriter, Read, Result, Seek, SeekFrom, Write}; +use std::path::PathBuf; const BLOCK_SIZE: usize = 4096; pub struct RegionFile { reader: BufReader, + writer: BufWriter, locations: Locations, #[allow(dead_code)] timestamps: Timestamps, } impl RegionFile { - pub fn new(reader: BufReader) -> Result { + pub fn new(path: &PathBuf) -> Result { + let fr = OpenOptions::new().read(true).open(path)?; + let fw = OpenOptions::new().write(true).open(path)?; + let mut reader = BufReader::with_capacity(BLOCK_SIZE, fr); + let writer = BufWriter::with_capacity(2 * BLOCK_SIZE, fw); + let mut locations_raw = [0u8; BLOCK_SIZE]; let mut timestamps_raw = [0u8; BLOCK_SIZE]; - let mut reader = reader; reader.read_exact(&mut locations_raw)?; reader.read_exact(&mut timestamps_raw)?; @@ -24,24 +30,17 @@ impl RegionFile { locations: Locations::from_bytes(&locations_raw), timestamps: Timestamps::from_bytes(×tamps_raw), reader, + writer, }) } - /// Writes a corrected version of the region file back to the disk - pub fn write(&self, writer: &mut BufWriter) -> Result<()> { - let location_bytes = self.locations.to_bytes(); - writer.write_all(&location_bytes.as_slice())?; - - writer.flush() - } - /// Returns the number of chunks in the file pub fn count_chunks(&self) -> usize { return self.locations.valid_entries().len(); } /// Scans the chunk entries for possible errors - pub fn scan_chunks(&mut self) -> Result { + pub fn scan_chunks(&mut self, fix: bool) -> Result { let mut statistic = ScanStatistics::new(); let entries = self.locations.valid_entries(); @@ -49,11 +48,21 @@ impl RegionFile { statistic.total_chunks = entries.len() as u64; for (offset, sections) in &entries { - self.reader - .seek(SeekFrom::Start(*offset as u64 * BLOCK_SIZE as u64))?; + let reader_offset = *offset as u64 * BLOCK_SIZE as u64; + self.reader.seek(SeekFrom::Start(reader_offset))?; + match self.read_chunk() { Ok(chunk) => { let chunk_sections = ((chunk.length + 4) as f64 / BLOCK_SIZE as f64).ceil(); + + if chunk.compression_type > 3 { + statistic.invalid_compression_method += 1; + if fix { + self.writer.seek(SeekFrom::Start(reader_offset + 4))?; + self.writer.write_u8(1)?; + } + } + if *sections != chunk_sections as u8 || chunk.length >= 1_048_576 { statistic.invalid_length += 1; corrected_entries.push((*offset, chunk_sections as u8)); @@ -62,12 +71,19 @@ impl RegionFile { } } Err(e) => { - println!("Failed to read chunk at {}: {}", offset, e); + log::error!("Failed to read chunk at {}: {}", offset, e); } } } self.locations.set_entries(corrected_entries); + if fix { + self.writer.seek(SeekFrom::Start(0))?; + self.writer + .write_all(self.locations.to_bytes().as_slice())?; + self.writer.flush()?; + } + Ok(statistic) } diff --git a/src/scan.rs b/src/scan.rs index 0edc919..9a2dd0b 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -5,13 +5,15 @@ use std::ops::Add; pub struct ScanStatistics { pub total_chunks: u64, pub invalid_length: u64, + pub invalid_compression_method: u64, } impl ScanStatistics { pub fn new() -> Self { Self { - invalid_length: 0, total_chunks: 0, + invalid_length: 0, + invalid_compression_method: 0, } } } @@ -22,6 +24,7 @@ impl Add for ScanStatistics { fn add(mut self, rhs: Self) -> Self::Output { self.invalid_length += rhs.invalid_length; self.total_chunks += rhs.total_chunks; + self.invalid_compression_method += rhs.invalid_compression_method; self } @@ -31,8 +34,8 @@ impl Display for ScanStatistics { fn fmt(&self, f: &mut Formatter<'_>) -> Result { write!( f, - "Total Chunks: {}\nChunks with invalid length: {}", - self.total_chunks, self.invalid_length + "Total Chunks: {}\nChunks with invalid length: {}\nChunks with invalid compression method: {}", + self.total_chunks, self.invalid_length, self.invalid_compression_method ) } } diff --git a/src/world_folder.rs b/src/world_folder.rs index b4c96a3..e9e62f9 100644 --- a/src/world_folder.rs +++ b/src/world_folder.rs @@ -1,14 +1,12 @@ use crate::region_file::RegionFile; use crate::scan::ScanStatistics; -use indicatif::{ProgressBar, ProgressStyle}; +use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle}; +use log::LevelFilter; use rayon::prelude::*; use std::fs; -use std::fs::{File, OpenOptions}; use std::io; -use std::io::{BufReader, BufWriter}; use std::ops::Add; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; pub struct WorldFolder { path: PathBuf, @@ -24,44 +22,47 @@ impl WorldFolder { let mut count = 0u64; for file in self.region_file_paths() { - let f = File::open(file)?; - let region_file = RegionFile::new(BufReader::new(f))?; + let region_file = RegionFile::new(&file)?; count += region_file.count_chunks() as u64; } Ok(count) } - pub fn scan_files(&self, fix: bool) -> io::Result<()> { + pub fn scan_files(&self, fix: bool) -> io::Result { let paths = self.region_file_paths(); - let bar = Arc::new(Mutex::new(ProgressBar::new(paths.len() as u64))); - bar.lock().unwrap().set_style( - ProgressStyle::default_bar().template("[{eta_precise}] {wide_bar} {pos}/{len} "), + let bar = ProgressBar::new(paths.len() as u64); + bar.set_style( + ProgressStyle::default_bar().template("\r[{eta_precise}] {wide_bar} {pos}/{len} "), ); + if log::max_level() == LevelFilter::Debug { + bar.set_draw_target(ProgressDrawTarget::hidden()) + } + bar.enable_steady_tick(1000); let statistic: ScanStatistics = paths .par_iter() - .filter_map(|file| { - let f = OpenOptions::new().read(true).open(file).ok()?; - let mut region_file = RegionFile::new(BufReader::new(f)).ok()?; + .filter_map(|path| { + log::debug!("Opening and scanning region file {:?}", path); + let mut region_file = RegionFile::new(path) + .map_err(|e| { + log::error!("Failed to open region file {:?}: {}", path, e); + + e + }) + .ok()?; - let result = region_file.scan_chunks().ok()?; - if fix { - let f = OpenOptions::new().write(true).open(file).ok()?; - let mut writer = BufWriter::new(f); - region_file.write(&mut writer).ok()?; - } - bar.lock().unwrap().inc(1); + let result = region_file.scan_chunks(fix).ok()?; + bar.inc(1); + log::debug!("Statistics for {:?}:\n{}", path, result); Some(result) }) .reduce(|| ScanStatistics::new(), |a, b| a.add(b)); - bar.lock().unwrap().finish_and_clear(); - - println!("{}", statistic); + bar.finish_and_clear(); - Ok(()) + Ok(statistic) } /// Returns a list of region file paths for the world folder