Optimize performance

- Update bdflib version
- Increase buffer size for data entries when decrypting
- Remove unnecessary read time report on decrypting
master
Trivernis 5 years ago
parent f7e51ebb3c
commit 78484ed656

6
Cargo.lock generated

@ -53,7 +53,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "bdflib" name = "bdflib"
version = "0.1.4" version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [ dependencies = [
"byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
@ -222,7 +222,7 @@ name = "destools"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bdflib 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "bdflib 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"cfb-mode 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "cfb-mode 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"des 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "des 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -815,7 +815,7 @@ dependencies = [
"checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
"checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" "checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
"checksum bdflib 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "27f26a69c90d4678e41a177a762b9916f36cc8ec5edb35efc845f1ee73607f73" "checksum bdflib 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ef9dbdad3b64cc71b2a43399e3ac54142bae88bc09d4e56fb9da9169b3f9ab9a"
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
"checksum blake2b_simd 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" "checksum blake2b_simd 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" "checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"

@ -139,14 +139,9 @@ fn decrypt(_opts: &Opts, args: &Decrypt) {
let data = fs::read(input).expect("Failed to read input file!"); let data = fs::read(input).expect("Failed to read input file!");
if let Some(input_checksum) = (args.clone()).input_checksum { if let Some(input_checksum) = (args.clone()).input_checksum {
tt.take("read-start");
let bin_content = fs::read(input_checksum).expect("Failed to read checksum file!"); let bin_content = fs::read(input_checksum).expect("Failed to read checksum file!");
let data_checksum = base64::decode(bin_content.as_slice()).unwrap(); let data_checksum = base64::decode(bin_content.as_slice()).unwrap();
println!(
"Reading took {:.2}s",
tt.since("read-start").unwrap().as_secs_f32()
);
if let Some(dict) = dictionary { if let Some(dict) = dictionary {
tt.take("decryption-start"); tt.take("decryption-start");
if let Some(dec_data) = decrypt_with_dictionary_file(dict, &data, &data_checksum) { if let Some(dec_data) = decrypt_with_dictionary_file(dict, &data, &data_checksum) {
@ -218,10 +213,9 @@ fn create_dictionary(_opts: &Opts, args: &CreateDictionary) {
pb.inc(); pb.inc();
} }
pb.finish(); pb.finish();
bdf_file.flush().expect("failed to flush the file data");
bdf_file bdf_file
.flush_writer() .finish()
.expect("failed to flush the file writer"); .expect("failed to finish the writing process");
}); });
tt.take("creation"); tt.take("creation");
@ -255,6 +249,9 @@ fn spinner(text: &str) -> Spinner {
Spinner::new(Spinners::Dots2, text.into()) Spinner::new(Spinners::Dots2, text.into())
} }
/// Decrypts the file using a bdf dictionary
/// The files content is read chunk by chunk to reduce the memory impact since dictionary
/// files tend to be several gigabytes in size
fn decrypt_with_dictionary_file( fn decrypt_with_dictionary_file(
filename: String, filename: String,
data: &Vec<u8>, data: &Vec<u8>,
@ -272,7 +269,7 @@ fn decrypt_with_dictionary_file(
chunk_count = meta.chunk_count; chunk_count = meta.chunk_count;
} }
let mut pb = ProgressBar::new(chunk_count as u64); let mut pb = ProgressBar::new(chunk_count as u64);
let (rx, tx) = sync_channel::<Vec<DataEntry>>(10); let (rx, tx) = sync_channel::<Vec<DataEntry>>(100);
let _handle = thread::spawn(move || { let _handle = thread::spawn(move || {
let mut lookup_table = HashLookupTable::new(HashMap::new()); let mut lookup_table = HashLookupTable::new(HashMap::new());
if let Ok(table) = bdf_file.read_lookup_table() { if let Ok(table) = bdf_file.read_lookup_table() {

Loading…
Cancel
Save