Add fixing of compression method

Signed-off-by: trivernis <trivernis@protonmail.com>
main
trivernis 4 years ago
parent 9d35c5772d
commit 2addee7348
Signed by: Trivernis
GPG Key ID: DFFFCC2C7A02DB45

89
Cargo.lock generated

@ -1,5 +1,14 @@
# This file is automatically @generated by Cargo. # This file is automatically @generated by Cargo.
# It is not intended for manual editing. # It is not intended for manual editing.
[[package]]
name = "aho-corasick"
version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "ansi_term" name = "ansi_term"
version = "0.11.0" version = "0.11.0"
@ -59,6 +68,17 @@ dependencies = [
"vec_map", "vec_map",
] ]
[[package]]
name = "colored"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
dependencies = [
"atty",
"lazy_static",
"winapi",
]
[[package]] [[package]]
name = "console" name = "console"
version = "0.12.0" version = "0.12.0"
@ -135,6 +155,19 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "env_logger"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
dependencies = [
"atty",
"humantime",
"log",
"regex",
"termcolor",
]
[[package]] [[package]]
name = "heck" name = "heck"
version = "0.3.1" version = "0.3.1"
@ -153,6 +186,15 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "humantime"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
dependencies = [
"quick-error",
]
[[package]] [[package]]
name = "indicatif" name = "indicatif"
version = "0.15.0" version = "0.15.0"
@ -177,12 +219,27 @@ version = "0.2.77"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235"
[[package]]
name = "log"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "maybe-uninit" name = "maybe-uninit"
version = "2.0.0" version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "memchr"
version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
[[package]] [[package]]
name = "memoffset" name = "memoffset"
version = "0.5.6" version = "0.5.6"
@ -194,10 +251,13 @@ dependencies = [
[[package]] [[package]]
name = "minecraft-regions-tool" name = "minecraft-regions-tool"
version = "0.2.0" version = "0.3.0"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"colored",
"env_logger",
"indicatif", "indicatif",
"log",
"rayon", "rayon",
"structopt", "structopt",
] ]
@ -251,6 +311,12 @@ dependencies = [
"unicode-xid", "unicode-xid",
] ]
[[package]]
name = "quick-error"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.7" version = "1.0.7"
@ -291,7 +357,10 @@ version = "1.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
dependencies = [ dependencies = [
"aho-corasick",
"memchr",
"regex-syntax", "regex-syntax",
"thread_local",
] ]
[[package]] [[package]]
@ -347,6 +416,15 @@ dependencies = [
"unicode-xid", "unicode-xid",
] ]
[[package]]
name = "termcolor"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
dependencies = [
"winapi-util",
]
[[package]] [[package]]
name = "terminal_size" name = "terminal_size"
version = "0.1.13" version = "0.1.13"
@ -375,6 +453,15 @@ dependencies = [
"unicode-width", "unicode-width",
] ]
[[package]]
name = "thread_local"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
dependencies = [
"lazy_static",
]
[[package]] [[package]]
name = "unicode-segmentation" name = "unicode-segmentation"
version = "1.6.0" version = "1.6.0"

@ -1,6 +1,6 @@
[package] [package]
name = "minecraft-regions-tool" name = "minecraft-regions-tool"
version = "0.2.0" version = "0.3.0"
authors = ["trivernis <trivernis@protonmail.com>"] authors = ["trivernis <trivernis@protonmail.com>"]
edition = "2018" edition = "2018"
license = "GPL-3.0" license = "GPL-3.0"
@ -15,3 +15,6 @@ byteorder = "1.3.4"
structopt = "0.3.18" structopt = "0.3.18"
rayon = "1.4.0" rayon = "1.4.0"
indicatif = "0.15.0" indicatif = "0.15.0"
log = "0.4.11"
env_logger ="0.7.1"
colored = "2.0.0"

@ -3,7 +3,7 @@ A utility to perform operations on minecraft region files
## Usage ## Usage
```sh ```
USAGE: USAGE:
minecraft-regions-tool <input> <SUBCOMMAND> minecraft-regions-tool <input> <SUBCOMMAND>

@ -1,3 +1,6 @@
use colored::*;
use env_logger::Env;
use log::Level;
use minecraft_regions_tool::world_folder::WorldFolder; use minecraft_regions_tool::world_folder::WorldFolder;
use std::path::PathBuf; use std::path::PathBuf;
use structopt::StructOpt; use structopt::StructOpt;
@ -8,6 +11,10 @@ struct Opt {
#[structopt(parse(from_os_str))] #[structopt(parse(from_os_str))]
input: PathBuf, input: PathBuf,
/// Forces verbose output
#[structopt(short, long)]
verbose: bool,
#[structopt(subcommand)] #[structopt(subcommand)]
sub_command: SubCommand, sub_command: SubCommand,
} }
@ -31,9 +38,50 @@ struct ScanOptions {
fn main() { fn main() {
let opt: Opt = Opt::from_args(); let opt: Opt = Opt::from_args();
build_logger(opt.verbose);
let world = WorldFolder::new(opt.input); let world = WorldFolder::new(opt.input);
match opt.sub_command { match opt.sub_command {
SubCommand::Count => println!("Chunk Count: {}", world.count_chunks().unwrap()), SubCommand::Count => log::info!("Chunk Count: {}", world.count_chunks().unwrap()),
SubCommand::Scan(opt) => world.scan_files(opt.fix).unwrap(), SubCommand::Scan(opt) => {
if opt.fix {
log::info!("Fixing fixable errors.");
}
log::info!("Scanning Region files for errors...");
log::info!("Scan Results:\n{}", world.scan_files(opt.fix).unwrap())
}
}
}
fn build_logger(verbose: bool) {
env_logger::Builder::from_env(Env::default().default_filter_or(if verbose {
"debug"
} else {
"info"
}))
.format(|buf, record| {
use std::io::Write;
let color = get_level_style(record.level());
writeln!(
buf,
"{}: {}",
record
.level()
.to_string()
.to_lowercase()
.as_str()
.color(color),
record.args()
)
})
.init();
}
fn get_level_style(level: Level) -> colored::Color {
match level {
Level::Trace => colored::Color::Magenta,
Level::Debug => colored::Color::Blue,
Level::Info => colored::Color::Green,
Level::Warn => colored::Color::Yellow,
Level::Error => colored::Color::Red,
} }
} }

@ -1,22 +1,28 @@
use crate::scan::ScanStatistics; use crate::scan::ScanStatistics;
use byteorder::{BigEndian, ByteOrder, ReadBytesExt}; use byteorder::{BigEndian, ByteOrder, ReadBytesExt, WriteBytesExt};
use std::fs::File; use std::fs::{File, OpenOptions};
use std::io::{BufReader, BufWriter, Read, Result, Seek, SeekFrom, Write}; use std::io::{BufReader, BufWriter, Read, Result, Seek, SeekFrom, Write};
use std::path::PathBuf;
const BLOCK_SIZE: usize = 4096; const BLOCK_SIZE: usize = 4096;
pub struct RegionFile { pub struct RegionFile {
reader: BufReader<File>, reader: BufReader<File>,
writer: BufWriter<File>,
locations: Locations, locations: Locations,
#[allow(dead_code)] #[allow(dead_code)]
timestamps: Timestamps, timestamps: Timestamps,
} }
impl RegionFile { impl RegionFile {
pub fn new(reader: BufReader<File>) -> Result<Self> { pub fn new(path: &PathBuf) -> Result<Self> {
let fr = OpenOptions::new().read(true).open(path)?;
let fw = OpenOptions::new().write(true).open(path)?;
let mut reader = BufReader::with_capacity(BLOCK_SIZE, fr);
let writer = BufWriter::with_capacity(2 * BLOCK_SIZE, fw);
let mut locations_raw = [0u8; BLOCK_SIZE]; let mut locations_raw = [0u8; BLOCK_SIZE];
let mut timestamps_raw = [0u8; BLOCK_SIZE]; let mut timestamps_raw = [0u8; BLOCK_SIZE];
let mut reader = reader;
reader.read_exact(&mut locations_raw)?; reader.read_exact(&mut locations_raw)?;
reader.read_exact(&mut timestamps_raw)?; reader.read_exact(&mut timestamps_raw)?;
@ -24,24 +30,17 @@ impl RegionFile {
locations: Locations::from_bytes(&locations_raw), locations: Locations::from_bytes(&locations_raw),
timestamps: Timestamps::from_bytes(&timestamps_raw), timestamps: Timestamps::from_bytes(&timestamps_raw),
reader, reader,
writer,
}) })
} }
/// Writes a corrected version of the region file back to the disk
pub fn write(&self, writer: &mut BufWriter<File>) -> Result<()> {
let location_bytes = self.locations.to_bytes();
writer.write_all(&location_bytes.as_slice())?;
writer.flush()
}
/// Returns the number of chunks in the file /// Returns the number of chunks in the file
pub fn count_chunks(&self) -> usize { pub fn count_chunks(&self) -> usize {
return self.locations.valid_entries().len(); return self.locations.valid_entries().len();
} }
/// Scans the chunk entries for possible errors /// Scans the chunk entries for possible errors
pub fn scan_chunks(&mut self) -> Result<ScanStatistics> { pub fn scan_chunks(&mut self, fix: bool) -> Result<ScanStatistics> {
let mut statistic = ScanStatistics::new(); let mut statistic = ScanStatistics::new();
let entries = self.locations.valid_entries(); let entries = self.locations.valid_entries();
@ -49,11 +48,21 @@ impl RegionFile {
statistic.total_chunks = entries.len() as u64; statistic.total_chunks = entries.len() as u64;
for (offset, sections) in &entries { for (offset, sections) in &entries {
self.reader let reader_offset = *offset as u64 * BLOCK_SIZE as u64;
.seek(SeekFrom::Start(*offset as u64 * BLOCK_SIZE as u64))?; self.reader.seek(SeekFrom::Start(reader_offset))?;
match self.read_chunk() { match self.read_chunk() {
Ok(chunk) => { Ok(chunk) => {
let chunk_sections = ((chunk.length + 4) as f64 / BLOCK_SIZE as f64).ceil(); let chunk_sections = ((chunk.length + 4) as f64 / BLOCK_SIZE as f64).ceil();
if chunk.compression_type > 3 {
statistic.invalid_compression_method += 1;
if fix {
self.writer.seek(SeekFrom::Start(reader_offset + 4))?;
self.writer.write_u8(1)?;
}
}
if *sections != chunk_sections as u8 || chunk.length >= 1_048_576 { if *sections != chunk_sections as u8 || chunk.length >= 1_048_576 {
statistic.invalid_length += 1; statistic.invalid_length += 1;
corrected_entries.push((*offset, chunk_sections as u8)); corrected_entries.push((*offset, chunk_sections as u8));
@ -62,12 +71,19 @@ impl RegionFile {
} }
} }
Err(e) => { Err(e) => {
println!("Failed to read chunk at {}: {}", offset, e); log::error!("Failed to read chunk at {}: {}", offset, e);
} }
} }
} }
self.locations.set_entries(corrected_entries); self.locations.set_entries(corrected_entries);
if fix {
self.writer.seek(SeekFrom::Start(0))?;
self.writer
.write_all(self.locations.to_bytes().as_slice())?;
self.writer.flush()?;
}
Ok(statistic) Ok(statistic)
} }

@ -5,13 +5,15 @@ use std::ops::Add;
pub struct ScanStatistics { pub struct ScanStatistics {
pub total_chunks: u64, pub total_chunks: u64,
pub invalid_length: u64, pub invalid_length: u64,
pub invalid_compression_method: u64,
} }
impl ScanStatistics { impl ScanStatistics {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
invalid_length: 0,
total_chunks: 0, total_chunks: 0,
invalid_length: 0,
invalid_compression_method: 0,
} }
} }
} }
@ -22,6 +24,7 @@ impl Add for ScanStatistics {
fn add(mut self, rhs: Self) -> Self::Output { fn add(mut self, rhs: Self) -> Self::Output {
self.invalid_length += rhs.invalid_length; self.invalid_length += rhs.invalid_length;
self.total_chunks += rhs.total_chunks; self.total_chunks += rhs.total_chunks;
self.invalid_compression_method += rhs.invalid_compression_method;
self self
} }
@ -31,8 +34,8 @@ impl Display for ScanStatistics {
fn fmt(&self, f: &mut Formatter<'_>) -> Result { fn fmt(&self, f: &mut Formatter<'_>) -> Result {
write!( write!(
f, f,
"Total Chunks: {}\nChunks with invalid length: {}", "Total Chunks: {}\nChunks with invalid length: {}\nChunks with invalid compression method: {}",
self.total_chunks, self.invalid_length self.total_chunks, self.invalid_length, self.invalid_compression_method
) )
} }
} }

@ -1,14 +1,12 @@
use crate::region_file::RegionFile; use crate::region_file::RegionFile;
use crate::scan::ScanStatistics; use crate::scan::ScanStatistics;
use indicatif::{ProgressBar, ProgressStyle}; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
use log::LevelFilter;
use rayon::prelude::*; use rayon::prelude::*;
use std::fs; use std::fs;
use std::fs::{File, OpenOptions};
use std::io; use std::io;
use std::io::{BufReader, BufWriter};
use std::ops::Add; use std::ops::Add;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::{Arc, Mutex};
pub struct WorldFolder { pub struct WorldFolder {
path: PathBuf, path: PathBuf,
@ -24,44 +22,47 @@ impl WorldFolder {
let mut count = 0u64; let mut count = 0u64;
for file in self.region_file_paths() { for file in self.region_file_paths() {
let f = File::open(file)?; let region_file = RegionFile::new(&file)?;
let region_file = RegionFile::new(BufReader::new(f))?;
count += region_file.count_chunks() as u64; count += region_file.count_chunks() as u64;
} }
Ok(count) Ok(count)
} }
pub fn scan_files(&self, fix: bool) -> io::Result<()> { pub fn scan_files(&self, fix: bool) -> io::Result<ScanStatistics> {
let paths = self.region_file_paths(); let paths = self.region_file_paths();
let bar = Arc::new(Mutex::new(ProgressBar::new(paths.len() as u64))); let bar = ProgressBar::new(paths.len() as u64);
bar.lock().unwrap().set_style( bar.set_style(
ProgressStyle::default_bar().template("[{eta_precise}] {wide_bar} {pos}/{len} "), ProgressStyle::default_bar().template("\r[{eta_precise}] {wide_bar} {pos}/{len} "),
); );
if log::max_level() == LevelFilter::Debug {
bar.set_draw_target(ProgressDrawTarget::hidden())
}
bar.enable_steady_tick(1000);
let statistic: ScanStatistics = paths let statistic: ScanStatistics = paths
.par_iter() .par_iter()
.filter_map(|file| { .filter_map(|path| {
let f = OpenOptions::new().read(true).open(file).ok()?; log::debug!("Opening and scanning region file {:?}", path);
let mut region_file = RegionFile::new(BufReader::new(f)).ok()?; let mut region_file = RegionFile::new(path)
.map_err(|e| {
log::error!("Failed to open region file {:?}: {}", path, e);
e
})
.ok()?;
let result = region_file.scan_chunks().ok()?; let result = region_file.scan_chunks(fix).ok()?;
if fix { bar.inc(1);
let f = OpenOptions::new().write(true).open(file).ok()?; log::debug!("Statistics for {:?}:\n{}", path, result);
let mut writer = BufWriter::new(f);
region_file.write(&mut writer).ok()?;
}
bar.lock().unwrap().inc(1);
Some(result) Some(result)
}) })
.reduce(|| ScanStatistics::new(), |a, b| a.add(b)); .reduce(|| ScanStatistics::new(), |a, b| a.add(b));
bar.lock().unwrap().finish_and_clear(); bar.finish_and_clear();
println!("{}", statistic);
Ok(()) Ok(statistic)
} }
/// Returns a list of region file paths for the world folder /// Returns a list of region file paths for the world folder

Loading…
Cancel
Save