Add scan commmand that also fixes lengths

The scan command looks for different section counts in the locations
table and optionally fixes it.

Signed-off-by: trivernis <trivernis@protonmail.com>
main 0.2.0
trivernis 4 years ago
parent a43301074d
commit 9d35c5772d
Signed by: Trivernis
GPG Key ID: DFFFCC2C7A02DB45

211
Cargo.lock generated

@ -20,6 +20,12 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "autocfg"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.2.1" version = "1.2.1"
@ -54,14 +60,81 @@ dependencies = [
] ]
[[package]] [[package]]
name = "cloudabi" name = "console"
version = "0.1.0" version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2"
dependencies = [ dependencies = [
"bitflags", "encode_unicode",
"lazy_static",
"libc",
"regex",
"terminal_size",
"termios",
"unicode-width",
"winapi",
"winapi-util",
]
[[package]]
name = "crossbeam-channel"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"
dependencies = [
"crossbeam-utils",
"maybe-uninit",
]
[[package]]
name = "crossbeam-deque"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"maybe-uninit",
] ]
[[package]]
name = "crossbeam-epoch"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"lazy_static",
"maybe-uninit",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg",
"cfg-if",
"lazy_static",
]
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]] [[package]]
name = "heck" name = "heck"
version = "0.3.1" version = "0.3.1"
@ -81,12 +154,15 @@ dependencies = [
] ]
[[package]] [[package]]
name = "instant" name = "indicatif"
version = "0.1.7" version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
dependencies = [ dependencies = [
"cfg-if", "console",
"lazy_static",
"number_prefix",
"regex",
] ]
[[package]] [[package]]
@ -102,21 +178,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235"
[[package]] [[package]]
name = "lock_api" name = "maybe-uninit"
version = "0.4.1" version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "memoffset"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
dependencies = [ dependencies = [
"scopeguard", "autocfg",
] ]
[[package]] [[package]]
name = "minecraft-regions-tool" name = "minecraft-regions-tool"
version = "0.1.0" version = "0.2.0"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"num_cpus", "indicatif",
"scheduled-thread-pool", "rayon",
"structopt", "structopt",
] ]
@ -131,30 +213,10 @@ dependencies = [
] ]
[[package]] [[package]]
name = "parking_lot" name = "number_prefix"
version = "0.11.0" version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
dependencies = [
"cfg-if",
"cloudabi",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
[[package]] [[package]]
name = "proc-macro-error" name = "proc-macro-error"
@ -199,31 +261,50 @@ dependencies = [
] ]
[[package]] [[package]]
name = "redox_syscall" name = "rayon"
version = "0.1.57" version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" checksum = "cfd016f0c045ad38b5251be2c9c0ab806917f82da4d36b2a327e5166adad9270"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]] [[package]]
name = "scheduled-thread-pool" name = "rayon-core"
version = "0.2.5" version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" checksum = "e8c4fec834fb6e6d2dd5eece3c7b432a52f0ba887cf40e595190c4107edc08bf"
dependencies = [ dependencies = [
"parking_lot", "crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"lazy_static",
"num_cpus",
] ]
[[package]] [[package]]
name = "scopeguard" name = "regex"
version = "1.1.0" version = "1.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
dependencies = [
"regex-syntax",
]
[[package]] [[package]]
name = "smallvec" name = "regex-syntax"
version = "1.4.2" version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "strsim" name = "strsim"
@ -266,6 +347,25 @@ dependencies = [
"unicode-xid", "unicode-xid",
] ]
[[package]]
name = "terminal_size"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "termios"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "textwrap" name = "textwrap"
version = "0.11.0" version = "0.11.0"
@ -321,6 +421,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "winapi-x86_64-pc-windows-gnu" name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0" version = "0.4.0"

@ -1,6 +1,6 @@
[package] [package]
name = "minecraft-regions-tool" name = "minecraft-regions-tool"
version = "0.1.0" version = "0.2.0"
authors = ["trivernis <trivernis@protonmail.com>"] authors = ["trivernis <trivernis@protonmail.com>"]
edition = "2018" edition = "2018"
license = "GPL-3.0" license = "GPL-3.0"
@ -13,5 +13,5 @@ repository = "https://github.com/Trivernis/minecraft-regions-tool"
[dependencies] [dependencies]
byteorder = "1.3.4" byteorder = "1.3.4"
structopt = "0.3.18" structopt = "0.3.18"
scheduled-thread-pool = "0.2.5" rayon = "1.4.0"
num_cpus = "1.13.0" indicatif = "0.15.0"

@ -17,4 +17,5 @@ ARGS:
SUBCOMMANDS: SUBCOMMANDS:
count Return the total number of chunks in the world count Return the total number of chunks in the world
help Prints this message or the help of the given subcommand(s) help Prints this message or the help of the given subcommand(s)
scan Scan for errors in the region files and optionally fix them
``` ```

@ -1,2 +1,3 @@
pub mod region_file; pub mod region_file;
pub mod scan;
pub mod world_folder; pub mod world_folder;

@ -17,6 +17,16 @@ struct Opt {
enum SubCommand { enum SubCommand {
/// Return the total number of chunks in the world /// Return the total number of chunks in the world
Count, Count,
/// Scan for errors in the region files and optionally fix them
Scan(ScanOptions),
}
#[derive(StructOpt, Debug)]
#[structopt()]
struct ScanOptions {
#[structopt(short, long)]
fix: bool,
} }
fn main() { fn main() {
@ -24,5 +34,6 @@ fn main() {
let world = WorldFolder::new(opt.input); let world = WorldFolder::new(opt.input);
match opt.sub_command { match opt.sub_command {
SubCommand::Count => println!("Chunk Count: {}", world.count_chunks().unwrap()), SubCommand::Count => println!("Chunk Count: {}", world.count_chunks().unwrap()),
SubCommand::Scan(opt) => world.scan_files(opt.fix).unwrap(),
} }
} }

@ -1,16 +1,19 @@
use byteorder::{BigEndian, ByteOrder}; use crate::scan::ScanStatistics;
use std::io::{Read, Result}; use byteorder::{BigEndian, ByteOrder, ReadBytesExt};
use std::fs::File;
use std::io::{BufReader, BufWriter, Read, Result, Seek, SeekFrom, Write};
const BLOCK_SIZE: usize = 4096; const BLOCK_SIZE: usize = 4096;
pub struct RegionFile { pub struct RegionFile {
reader: Box<dyn Read>, reader: BufReader<File>,
locations: Locations, locations: Locations,
#[allow(dead_code)]
timestamps: Timestamps, timestamps: Timestamps,
} }
impl RegionFile { impl RegionFile {
pub fn new(reader: Box<dyn Read>) -> Result<Self> { pub fn new(reader: BufReader<File>) -> Result<Self> {
let mut locations_raw = [0u8; BLOCK_SIZE]; let mut locations_raw = [0u8; BLOCK_SIZE];
let mut timestamps_raw = [0u8; BLOCK_SIZE]; let mut timestamps_raw = [0u8; BLOCK_SIZE];
let mut reader = reader; let mut reader = reader;
@ -24,20 +27,67 @@ impl RegionFile {
}) })
} }
/// Writes a corrected version of the region file back to the disk
pub fn write(&self, writer: &mut BufWriter<File>) -> Result<()> {
let location_bytes = self.locations.to_bytes();
writer.write_all(&location_bytes.as_slice())?;
writer.flush()
}
/// Returns the number of chunks in the file /// Returns the number of chunks in the file
pub fn count_chunks(&self) -> usize { pub fn count_chunks(&self) -> usize {
let mut count = 0; return self.locations.valid_entries().len();
for x in 0..32 { }
for z in 0..32 {
if !(self.locations.get_chunk_offset(x, z) == Some(0) /// Scans the chunk entries for possible errors
&& self.locations.get_chunk_sectors(x, z) == Some(0)) pub fn scan_chunks(&mut self) -> Result<ScanStatistics> {
{ let mut statistic = ScanStatistics::new();
count += 1;
let entries = self.locations.valid_entries();
let mut corrected_entries = Vec::new();
statistic.total_chunks = entries.len() as u64;
for (offset, sections) in &entries {
self.reader
.seek(SeekFrom::Start(*offset as u64 * BLOCK_SIZE as u64))?;
match self.read_chunk() {
Ok(chunk) => {
let chunk_sections = ((chunk.length + 4) as f64 / BLOCK_SIZE as f64).ceil();
if *sections != chunk_sections as u8 || chunk.length >= 1_048_576 {
statistic.invalid_length += 1;
corrected_entries.push((*offset, chunk_sections as u8));
} else {
corrected_entries.push((*offset, *sections));
}
}
Err(e) => {
println!("Failed to read chunk at {}: {}", offset, e);
} }
} }
} }
self.locations.set_entries(corrected_entries);
return count; Ok(statistic)
}
/// Reads a chunk at the current location
fn read_chunk(&mut self) -> Result<Chunk> {
let mut length_raw = [0u8; 4];
self.reader.read_exact(&mut length_raw)?;
let length = BigEndian::read_u32(&length_raw);
let compression_type = self.reader.read_u8()?;
if length > 0 {
self.reader.seek(SeekFrom::Current((length - 1) as i64))?;
} else {
self.reader.seek(SeekFrom::Current((length) as i64))?;
}
Ok(Chunk {
length,
compression_type,
})
} }
} }
@ -51,8 +101,8 @@ impl Locations {
let mut locations = Vec::new(); let mut locations = Vec::new();
for i in (0..BLOCK_SIZE - 1).step_by(4) { for i in (0..BLOCK_SIZE - 1).step_by(4) {
let mut offset = BigEndian::read_u32(&bytes[i..i + 4]); let offset_raw = [0u8, bytes[i], bytes[i + 1], bytes[i + 2]];
offset = offset >> 1; let offset = BigEndian::read_u32(&offset_raw);
let count = bytes[i + 3]; let count = bytes[i + 3];
locations.push((offset, count)); locations.push((offset, count));
} }
@ -60,6 +110,20 @@ impl Locations {
Self { inner: locations } Self { inner: locations }
} }
/// Returns the byte representation of the locations table
pub fn to_bytes(&self) -> Vec<u8> {
let mut bytes = Vec::new();
for (offset, sections) in &self.inner {
let mut offset_raw = [0u8; 4];
BigEndian::write_u32(&mut offset_raw, *offset);
bytes.append(&mut offset_raw[1..4].to_vec());
bytes.push(*sections);
}
bytes
}
/// Returns the offset of a chunk /// Returns the offset of a chunk
pub fn get_chunk_offset(&self, x: usize, z: usize) -> Option<u32> { pub fn get_chunk_offset(&self, x: usize, z: usize) -> Option<u32> {
let index = x % 32 + (z % 32) * 32; let index = x % 32 + (z % 32) * 32;
@ -71,6 +135,19 @@ impl Locations {
let index = x % 32 + (z % 32) * 32; let index = x % 32 + (z % 32) * 32;
self.inner.get(index).map(|e| (*e).1) self.inner.get(index).map(|e| (*e).1)
} }
/// Returns chunk entry list
pub fn valid_entries(&self) -> Vec<(u32, u8)> {
self.inner
.iter()
.filter_map(|e| if (*e).0 >= 2 { Some(*e) } else { None })
.collect()
}
/// Replaces the entry list with a new one
pub fn set_entries(&mut self, entries: Vec<(u32, u8)>) {
self.inner = entries;
}
} }
#[derive(Debug)] #[derive(Debug)]
@ -89,3 +166,9 @@ impl Timestamps {
Self { inner: timestamps } Self { inner: timestamps }
} }
} }
#[derive(Debug)]
pub struct Chunk {
pub length: u32,
pub compression_type: u8,
}

@ -0,0 +1,38 @@
use std::fmt::{Display, Formatter, Result};
use std::ops::Add;
#[derive(Clone, Debug)]
pub struct ScanStatistics {
pub total_chunks: u64,
pub invalid_length: u64,
}
impl ScanStatistics {
pub fn new() -> Self {
Self {
invalid_length: 0,
total_chunks: 0,
}
}
}
impl Add for ScanStatistics {
type Output = Self;
fn add(mut self, rhs: Self) -> Self::Output {
self.invalid_length += rhs.invalid_length;
self.total_chunks += rhs.total_chunks;
self
}
}
impl Display for ScanStatistics {
fn fmt(&self, f: &mut Formatter<'_>) -> Result {
write!(
f,
"Total Chunks: {}\nChunks with invalid length: {}",
self.total_chunks, self.invalid_length
)
}
}

@ -1,10 +1,14 @@
use crate::region_file::RegionFile; use crate::region_file::RegionFile;
use std::ffi::OsStr; use crate::scan::ScanStatistics;
use indicatif::{ProgressBar, ProgressStyle};
use rayon::prelude::*;
use std::fs; use std::fs;
use std::fs::File; use std::fs::{File, OpenOptions};
use std::io; use std::io;
use std::io::BufReader; use std::io::{BufReader, BufWriter};
use std::ops::Add;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::{Arc, Mutex};
pub struct WorldFolder { pub struct WorldFolder {
path: PathBuf, path: PathBuf,
@ -15,19 +19,58 @@ impl WorldFolder {
Self { path } Self { path }
} }
/// Counts all chunks of a world
pub fn count_chunks(&self) -> io::Result<u64> { pub fn count_chunks(&self) -> io::Result<u64> {
let mut count = 0u64; let mut count = 0u64;
let region_file_path = self.path.join(PathBuf::from("region"));
for file in fs::read_dir(region_file_path)? { for file in self.region_file_paths() {
let file_path = file?.path(); let f = File::open(file)?;
if file_path.extension() == Some(OsStr::new("mca")) { let region_file = RegionFile::new(BufReader::new(f))?;
let f = File::open(file_path)?; count += region_file.count_chunks() as u64;
let region_file = RegionFile::new(Box::new(BufReader::new(f)))?;
count += region_file.count_chunks() as u64;
}
} }
Ok(count) Ok(count)
} }
pub fn scan_files(&self, fix: bool) -> io::Result<()> {
let paths = self.region_file_paths();
let bar = Arc::new(Mutex::new(ProgressBar::new(paths.len() as u64)));
bar.lock().unwrap().set_style(
ProgressStyle::default_bar().template("[{eta_precise}] {wide_bar} {pos}/{len} "),
);
let statistic: ScanStatistics = paths
.par_iter()
.filter_map(|file| {
let f = OpenOptions::new().read(true).open(file).ok()?;
let mut region_file = RegionFile::new(BufReader::new(f)).ok()?;
let result = region_file.scan_chunks().ok()?;
if fix {
let f = OpenOptions::new().write(true).open(file).ok()?;
let mut writer = BufWriter::new(f);
region_file.write(&mut writer).ok()?;
}
bar.lock().unwrap().inc(1);
Some(result)
})
.reduce(|| ScanStatistics::new(), |a, b| a.add(b));
bar.lock().unwrap().finish_and_clear();
println!("{}", statistic);
Ok(())
}
/// Returns a list of region file paths for the world folder
fn region_file_paths(&self) -> Vec<PathBuf> {
let region_file_path = self.path.join(PathBuf::from("region"));
fs::read_dir(region_file_path)
.unwrap()
.filter_map(|e| e.ok().map(|e| e.path()))
.collect()
}
} }

Loading…
Cancel
Save