From 4842b9f34668033dfc3ae07fc9fada2c04193c6e Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 27 Sep 2020 14:24:27 +0200 Subject: [PATCH] Fix shifting of chunks to occupy unused space Signed-off-by: trivernis --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/region_file.rs | 85 ++++++++++++++++++++++++++++++++++++++------- src/scan.rs | 11 ++++-- src/world_folder.rs | 5 +++ 5 files changed, 88 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b02cbeb..838f155 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -307,7 +307,7 @@ dependencies = [ [[package]] name = "minecraft-regions-tool" -version = "0.5.0" +version = "0.5.1" dependencies = [ "byteorder", "colored", diff --git a/Cargo.toml b/Cargo.toml index d20cc67..5331e88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "minecraft-regions-tool" -version = "0.5.0" +version = "0.5.1" authors = ["trivernis "] edition = "2018" license = "GPL-3.0" diff --git a/src/region_file.rs b/src/region_file.rs index fb94360..96b2845 100644 --- a/src/region_file.rs +++ b/src/region_file.rs @@ -46,6 +46,7 @@ impl RegionFile { /// Scans the chunk entries for possible errors pub fn scan_chunks(&mut self, options: &Arc) -> Result { let mut statistic = ScanStatistics::new(); + let mut shift_operations: Vec<(usize, isize)> = Vec::new(); let mut entries = self.locations.valid_entries_enumerate(); entries.sort_by(|(_, (a, _)), (_, (b, _))| { @@ -58,6 +59,8 @@ impl RegionFile { } }); statistic.total_chunks = entries.len() as u64; + let mut previous_offset = 2; + let mut previous_sections = 0; for (index, (offset, sections)) in entries { let reader_offset = offset as u64 * BLOCK_SIZE as u64; @@ -65,16 +68,41 @@ impl RegionFile { match Chunk::from_buf_reader(&mut self.reader) { Ok(chunk) => { - self.scan_chunk(index, offset, sections, chunk, &mut statistic, options)?; + let exists = + self.scan_chunk(index, offset, sections, chunk, &mut statistic, options)?; + if !exists && options.fix { + shift_operations + .push((offset as usize + sections as usize, -(sections as isize))) + } } Err(e) => { statistic.failed_to_read += 1; + if options.fix_delete { + self.delete_chunk(index)?; + } log::error!("Failed to read chunk at {}: {}", offset, e); } } + let offset_diff = offset - (previous_offset + previous_sections); + if offset_diff > 0 { + statistic.unused_space += (BLOCK_SIZE * offset_diff as usize) as u64; + if options.fix { + shift_operations.push((offset as usize, -(offset_diff as isize))); + } + } + previous_offset = offset; + previous_sections = sections as u32; } if options.fix || options.fix_delete { + let mut shifted = 0isize; + for (offset, amount) in shift_operations { + let offset = (offset as isize + shifted) as usize; + self.shift_right(offset, amount)?; + self.locations.shift_entries(offset as u32, amount as i32); + shifted += amount; + } + statistic.shrunk_size = self.locations.estimated_size(); self.writer.seek(SeekFrom::Start(0))?; self.writer .write_all(self.locations.to_bytes().as_slice())?; @@ -93,7 +121,7 @@ impl RegionFile { mut chunk: Chunk, statistic: &mut ScanStatistics, options: &Arc, - ) -> Result<()> { + ) -> Result { let chunk_sections = ((chunk.length + 4) as f64 / BLOCK_SIZE as f64).ceil(); let reader_offset = offset as u64 * BLOCK_SIZE as u64; @@ -120,32 +148,45 @@ impl RegionFile { statistic.missing_nbt += 1; } } - self.delete_chunk(index)?; + if options.fix_delete { + self.delete_chunk(index)?; + return Ok(false); + } } } if sections != chunk_sections as u8 || chunk.length >= 1_048_576 { statistic.invalid_length += 1; - self.locations - .replace_entry_unchecked(index, (offset, chunk_sections as u8)); + if options.fix { + self.locations + .replace_entry_unchecked(index, (offset, chunk_sections as u8)); + } } - Ok(()) + Ok(true) } /// Deletes a chunk and shifts all other chunks pub fn delete_chunk(&mut self, index: usize) -> Result<()> { let (offset, sections) = self.locations.get_chunk_entry_unchecked(index); - self.reader.seek(SeekFrom::Start( - (offset as usize * BLOCK_SIZE + sections as usize * BLOCK_SIZE) as u64, - ))?; - self.writer - .seek(SeekFrom::Start((offset as usize * BLOCK_SIZE) as u64))?; + log::debug!( "Shifting chunk entries starting from {} by {} to the left", offset, sections as u32 ); + + self.locations.delete_chunk_entry_unchecked(index); + Ok(()) + } + + /// Shifts the file from the `offset` position `amount` blocks to the right + pub fn shift_right(&mut self, offset: usize, amount: isize) -> Result<()> { + self.reader + .seek(SeekFrom::Start((offset * BLOCK_SIZE) as u64))?; + self.writer.seek(SeekFrom::Start( + ((offset as isize + amount) as usize * BLOCK_SIZE) as u64, + ))?; loop { let mut buf = [0u8; BLOCK_SIZE]; let read = self.reader.read(&mut buf)?; @@ -154,8 +195,6 @@ impl RegionFile { break; } } - self.locations.delete_chunk_entry_unchecked(index); - self.locations.shift_entries(offset, -(sections as i32)); Ok(()) } @@ -221,6 +260,26 @@ impl Locations { .collect() } + /// Returns the estimated of all chunks combined including the header + pub fn estimated_size(&self) -> u64 { + let largest = self + .inner + .iter() + .max_by(|(a, _), (b, _)| { + if a > b { + Ordering::Greater + } else if a < b { + Ordering::Less + } else { + Ordering::Equal + } + }) + .cloned() + .unwrap_or((2, 0)); + + (largest.0 as u64 + largest.1 as u64) * BLOCK_SIZE as u64 + } + /// Replaces an entry with a new one. Panics if the index doesn't exist pub fn replace_entry_unchecked(&mut self, index: usize, entry: (u32, u8)) { self.inner[index] = entry; diff --git a/src/scan.rs b/src/scan.rs index 98877cb..2cfb628 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -10,6 +10,8 @@ pub struct ScanStatistics { pub corrupted_nbt: u64, pub failed_to_read: u64, pub corrupted_compression: u64, + pub shrunk_size: u64, + pub unused_space: u64, } impl ScanStatistics { @@ -22,6 +24,8 @@ impl ScanStatistics { corrupted_nbt: 0, corrupted_compression: 0, failed_to_read: 0, + shrunk_size: 0, + unused_space: 0, } } } @@ -37,6 +41,7 @@ impl Add for ScanStatistics { self.missing_nbt += rhs.missing_nbt; self.corrupted_compression += rhs.corrupted_compression; self.corrupted_nbt += rhs.corrupted_nbt; + self.unused_space += rhs.unused_space; self } @@ -53,14 +58,16 @@ impl Display for ScanStatistics { Chunks with invalid compression method: {} Chunks with missing nbt data: {} Chunks with corrupted nbt data: {} - Chunks with corrupted compressed data {}", + Chunks with corrupted compressed data: {} + Unused space: {} KiB", self.total_chunks, self.failed_to_read, self.invalid_length, self.invalid_compression_method, self.missing_nbt, self.corrupted_nbt, - self.corrupted_compression + self.corrupted_compression, + self.unused_space / 1024, ) } } diff --git a/src/world_folder.rs b/src/world_folder.rs index d2d3714..703271b 100644 --- a/src/world_folder.rs +++ b/src/world_folder.rs @@ -5,6 +5,7 @@ use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle}; use log::LevelFilter; use rayon::prelude::*; use std::fs; +use std::fs::OpenOptions; use std::io; use std::ops::Add; use std::path::PathBuf; @@ -62,6 +63,10 @@ impl WorldFolder { .ok()?; let result = region_file.scan_chunks(&options).ok()?; + if options.fix && result.shrunk_size > 0 { + let f = OpenOptions::new().read(true).write(true).open(path).ok()?; + f.set_len(result.shrunk_size).ok()?; + } bar.inc(1); log::debug!("Statistics for {:?}:\n{}", path, result);