Add fixing of compression method

Signed-off-by: trivernis <trivernis@protonmail.com>
main
trivernis 4 years ago
parent 9d35c5772d
commit 2addee7348
Signed by: Trivernis
GPG Key ID: DFFFCC2C7A02DB45

89
Cargo.lock generated

@ -1,5 +1,14 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "aho-corasick"
version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86"
dependencies = [
"memchr",
]
[[package]]
name = "ansi_term"
version = "0.11.0"
@ -59,6 +68,17 @@ dependencies = [
"vec_map",
]
[[package]]
name = "colored"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
dependencies = [
"atty",
"lazy_static",
"winapi",
]
[[package]]
name = "console"
version = "0.12.0"
@ -135,6 +155,19 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "env_logger"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
dependencies = [
"atty",
"humantime",
"log",
"regex",
"termcolor",
]
[[package]]
name = "heck"
version = "0.3.1"
@ -153,6 +186,15 @@ dependencies = [
"libc",
]
[[package]]
name = "humantime"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
dependencies = [
"quick-error",
]
[[package]]
name = "indicatif"
version = "0.15.0"
@ -177,12 +219,27 @@ version = "0.2.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235"
[[package]]
name = "log"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
dependencies = [
"cfg-if",
]
[[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "memchr"
version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
[[package]]
name = "memoffset"
version = "0.5.6"
@ -194,10 +251,13 @@ dependencies = [
[[package]]
name = "minecraft-regions-tool"
version = "0.2.0"
version = "0.3.0"
dependencies = [
"byteorder",
"colored",
"env_logger",
"indicatif",
"log",
"rayon",
"structopt",
]
@ -251,6 +311,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "quick-error"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quote"
version = "1.0.7"
@ -291,7 +357,10 @@ version = "1.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
"thread_local",
]
[[package]]
@ -347,6 +416,15 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "termcolor"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
dependencies = [
"winapi-util",
]
[[package]]
name = "terminal_size"
version = "0.1.13"
@ -375,6 +453,15 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "thread_local"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
dependencies = [
"lazy_static",
]
[[package]]
name = "unicode-segmentation"
version = "1.6.0"

@ -1,6 +1,6 @@
[package]
name = "minecraft-regions-tool"
version = "0.2.0"
version = "0.3.0"
authors = ["trivernis <trivernis@protonmail.com>"]
edition = "2018"
license = "GPL-3.0"
@ -15,3 +15,6 @@ byteorder = "1.3.4"
structopt = "0.3.18"
rayon = "1.4.0"
indicatif = "0.15.0"
log = "0.4.11"
env_logger ="0.7.1"
colored = "2.0.0"

@ -3,7 +3,7 @@ A utility to perform operations on minecraft region files
## Usage
```sh
```
USAGE:
minecraft-regions-tool <input> <SUBCOMMAND>

@ -1,3 +1,6 @@
use colored::*;
use env_logger::Env;
use log::Level;
use minecraft_regions_tool::world_folder::WorldFolder;
use std::path::PathBuf;
use structopt::StructOpt;
@ -8,6 +11,10 @@ struct Opt {
#[structopt(parse(from_os_str))]
input: PathBuf,
/// Forces verbose output
#[structopt(short, long)]
verbose: bool,
#[structopt(subcommand)]
sub_command: SubCommand,
}
@ -31,9 +38,50 @@ struct ScanOptions {
fn main() {
let opt: Opt = Opt::from_args();
build_logger(opt.verbose);
let world = WorldFolder::new(opt.input);
match opt.sub_command {
SubCommand::Count => println!("Chunk Count: {}", world.count_chunks().unwrap()),
SubCommand::Scan(opt) => world.scan_files(opt.fix).unwrap(),
SubCommand::Count => log::info!("Chunk Count: {}", world.count_chunks().unwrap()),
SubCommand::Scan(opt) => {
if opt.fix {
log::info!("Fixing fixable errors.");
}
log::info!("Scanning Region files for errors...");
log::info!("Scan Results:\n{}", world.scan_files(opt.fix).unwrap())
}
}
}
fn build_logger(verbose: bool) {
env_logger::Builder::from_env(Env::default().default_filter_or(if verbose {
"debug"
} else {
"info"
}))
.format(|buf, record| {
use std::io::Write;
let color = get_level_style(record.level());
writeln!(
buf,
"{}: {}",
record
.level()
.to_string()
.to_lowercase()
.as_str()
.color(color),
record.args()
)
})
.init();
}
fn get_level_style(level: Level) -> colored::Color {
match level {
Level::Trace => colored::Color::Magenta,
Level::Debug => colored::Color::Blue,
Level::Info => colored::Color::Green,
Level::Warn => colored::Color::Yellow,
Level::Error => colored::Color::Red,
}
}

@ -1,22 +1,28 @@
use crate::scan::ScanStatistics;
use byteorder::{BigEndian, ByteOrder, ReadBytesExt};
use std::fs::File;
use byteorder::{BigEndian, ByteOrder, ReadBytesExt, WriteBytesExt};
use std::fs::{File, OpenOptions};
use std::io::{BufReader, BufWriter, Read, Result, Seek, SeekFrom, Write};
use std::path::PathBuf;
const BLOCK_SIZE: usize = 4096;
pub struct RegionFile {
reader: BufReader<File>,
writer: BufWriter<File>,
locations: Locations,
#[allow(dead_code)]
timestamps: Timestamps,
}
impl RegionFile {
pub fn new(reader: BufReader<File>) -> Result<Self> {
pub fn new(path: &PathBuf) -> Result<Self> {
let fr = OpenOptions::new().read(true).open(path)?;
let fw = OpenOptions::new().write(true).open(path)?;
let mut reader = BufReader::with_capacity(BLOCK_SIZE, fr);
let writer = BufWriter::with_capacity(2 * BLOCK_SIZE, fw);
let mut locations_raw = [0u8; BLOCK_SIZE];
let mut timestamps_raw = [0u8; BLOCK_SIZE];
let mut reader = reader;
reader.read_exact(&mut locations_raw)?;
reader.read_exact(&mut timestamps_raw)?;
@ -24,24 +30,17 @@ impl RegionFile {
locations: Locations::from_bytes(&locations_raw),
timestamps: Timestamps::from_bytes(&timestamps_raw),
reader,
writer,
})
}
/// Writes a corrected version of the region file back to the disk
pub fn write(&self, writer: &mut BufWriter<File>) -> Result<()> {
let location_bytes = self.locations.to_bytes();
writer.write_all(&location_bytes.as_slice())?;
writer.flush()
}
/// Returns the number of chunks in the file
pub fn count_chunks(&self) -> usize {
return self.locations.valid_entries().len();
}
/// Scans the chunk entries for possible errors
pub fn scan_chunks(&mut self) -> Result<ScanStatistics> {
pub fn scan_chunks(&mut self, fix: bool) -> Result<ScanStatistics> {
let mut statistic = ScanStatistics::new();
let entries = self.locations.valid_entries();
@ -49,11 +48,21 @@ impl RegionFile {
statistic.total_chunks = entries.len() as u64;
for (offset, sections) in &entries {
self.reader
.seek(SeekFrom::Start(*offset as u64 * BLOCK_SIZE as u64))?;
let reader_offset = *offset as u64 * BLOCK_SIZE as u64;
self.reader.seek(SeekFrom::Start(reader_offset))?;
match self.read_chunk() {
Ok(chunk) => {
let chunk_sections = ((chunk.length + 4) as f64 / BLOCK_SIZE as f64).ceil();
if chunk.compression_type > 3 {
statistic.invalid_compression_method += 1;
if fix {
self.writer.seek(SeekFrom::Start(reader_offset + 4))?;
self.writer.write_u8(1)?;
}
}
if *sections != chunk_sections as u8 || chunk.length >= 1_048_576 {
statistic.invalid_length += 1;
corrected_entries.push((*offset, chunk_sections as u8));
@ -62,12 +71,19 @@ impl RegionFile {
}
}
Err(e) => {
println!("Failed to read chunk at {}: {}", offset, e);
log::error!("Failed to read chunk at {}: {}", offset, e);
}
}
}
self.locations.set_entries(corrected_entries);
if fix {
self.writer.seek(SeekFrom::Start(0))?;
self.writer
.write_all(self.locations.to_bytes().as_slice())?;
self.writer.flush()?;
}
Ok(statistic)
}

@ -5,13 +5,15 @@ use std::ops::Add;
pub struct ScanStatistics {
pub total_chunks: u64,
pub invalid_length: u64,
pub invalid_compression_method: u64,
}
impl ScanStatistics {
pub fn new() -> Self {
Self {
invalid_length: 0,
total_chunks: 0,
invalid_length: 0,
invalid_compression_method: 0,
}
}
}
@ -22,6 +24,7 @@ impl Add for ScanStatistics {
fn add(mut self, rhs: Self) -> Self::Output {
self.invalid_length += rhs.invalid_length;
self.total_chunks += rhs.total_chunks;
self.invalid_compression_method += rhs.invalid_compression_method;
self
}
@ -31,8 +34,8 @@ impl Display for ScanStatistics {
fn fmt(&self, f: &mut Formatter<'_>) -> Result {
write!(
f,
"Total Chunks: {}\nChunks with invalid length: {}",
self.total_chunks, self.invalid_length
"Total Chunks: {}\nChunks with invalid length: {}\nChunks with invalid compression method: {}",
self.total_chunks, self.invalid_length, self.invalid_compression_method
)
}
}

@ -1,14 +1,12 @@
use crate::region_file::RegionFile;
use crate::scan::ScanStatistics;
use indicatif::{ProgressBar, ProgressStyle};
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
use log::LevelFilter;
use rayon::prelude::*;
use std::fs;
use std::fs::{File, OpenOptions};
use std::io;
use std::io::{BufReader, BufWriter};
use std::ops::Add;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
pub struct WorldFolder {
path: PathBuf,
@ -24,44 +22,47 @@ impl WorldFolder {
let mut count = 0u64;
for file in self.region_file_paths() {
let f = File::open(file)?;
let region_file = RegionFile::new(BufReader::new(f))?;
let region_file = RegionFile::new(&file)?;
count += region_file.count_chunks() as u64;
}
Ok(count)
}
pub fn scan_files(&self, fix: bool) -> io::Result<()> {
pub fn scan_files(&self, fix: bool) -> io::Result<ScanStatistics> {
let paths = self.region_file_paths();
let bar = Arc::new(Mutex::new(ProgressBar::new(paths.len() as u64)));
bar.lock().unwrap().set_style(
ProgressStyle::default_bar().template("[{eta_precise}] {wide_bar} {pos}/{len} "),
let bar = ProgressBar::new(paths.len() as u64);
bar.set_style(
ProgressStyle::default_bar().template("\r[{eta_precise}] {wide_bar} {pos}/{len} "),
);
if log::max_level() == LevelFilter::Debug {
bar.set_draw_target(ProgressDrawTarget::hidden())
}
bar.enable_steady_tick(1000);
let statistic: ScanStatistics = paths
.par_iter()
.filter_map(|file| {
let f = OpenOptions::new().read(true).open(file).ok()?;
let mut region_file = RegionFile::new(BufReader::new(f)).ok()?;
.filter_map(|path| {
log::debug!("Opening and scanning region file {:?}", path);
let mut region_file = RegionFile::new(path)
.map_err(|e| {
log::error!("Failed to open region file {:?}: {}", path, e);
e
})
.ok()?;
let result = region_file.scan_chunks().ok()?;
if fix {
let f = OpenOptions::new().write(true).open(file).ok()?;
let mut writer = BufWriter::new(f);
region_file.write(&mut writer).ok()?;
}
bar.lock().unwrap().inc(1);
let result = region_file.scan_chunks(fix).ok()?;
bar.inc(1);
log::debug!("Statistics for {:?}:\n{}", path, result);
Some(result)
})
.reduce(|| ScanStatistics::new(), |a, b| a.add(b));
bar.lock().unwrap().finish_and_clear();
println!("{}", statistic);
bar.finish_and_clear();
Ok(())
Ok(statistic)
}
/// Returns a list of region file paths for the world folder

Loading…
Cancel
Save