From 63ea60b10afc3d53dd2ef42f8f1c8339284af153 Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 16 Dec 2020 16:14:20 +0100 Subject: [PATCH] Add conversion of images when configured Signed-off-by: trivernis --- Cargo.lock | 253 ++++++++++++++++++++++++++- Cargo.toml | 5 +- README.md | 19 ++ src/elements/mod.rs | 75 +++++++- src/format/html/to_html.rs | 24 +-- src/main.rs | 6 +- src/parser/inline.rs | 15 +- src/parser/line.rs | 2 - src/parser/mod.rs | 4 +- src/references/bibliography.rs | 3 +- src/references/configuration/keys.rs | 5 + src/references/glossary.rs | 15 +- src/references/placeholders.rs | 2 +- src/utils/caching.rs | 9 +- src/utils/downloads.rs | 25 +-- src/utils/image_converting.rs | 176 +++++++++++++++++++ src/utils/mod.rs | 1 + 17 files changed, 568 insertions(+), 71 deletions(-) create mode 100644 src/utils/image_converting.rs diff --git a/Cargo.lock b/Cargo.lock index e412a83..edf87b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aho-corasick" version = "0.7.15" @@ -89,7 +95,7 @@ dependencies = [ "addr2line", "cfg-if 1.0.0", "libc", - "miniz_oxide", + "miniz_oxide 0.4.3", "object", "rustc-demangle", ] @@ -127,12 +133,13 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bibliographix" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39034545c510b822e3e5bd76147f869bae617d52961add6313ad0696084585c1" +checksum = "bef9342b1214c0ff300bb812af4c9a35e0ec1351037a2a3e8595f24483a953d3" dependencies = [ "chrono", "chrono-english", + "parking_lot", "toml", ] @@ -163,12 +170,27 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +[[package]] +name = "bytemuck" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41aa2ec95ca3b5c54cf73c91acf06d24f4495d5f1b1c12506ae3483d646177ac" + [[package]] name = "byteorder" version = "1.3.4" @@ -253,6 +275,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colored" version = "1.9.3" @@ -318,6 +346,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + [[package]] name = "crc32fast" version = "1.2.1" @@ -419,6 +453,16 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", +] + [[package]] name = "derive_builder" version = "0.7.2" @@ -444,6 +488,15 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "dirs" version = "2.0.2" @@ -555,7 +608,7 @@ dependencies = [ "cfg-if 1.0.0", "crc32fast", "libc", - "miniz_oxide", + "miniz_oxide 0.4.3", ] [[package]] @@ -681,6 +734,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check 0.9.2", +] + [[package]] name = "getrandom" version = "0.1.15" @@ -702,6 +765,16 @@ dependencies = [ "regex", ] +[[package]] +name = "gif" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02efba560f227847cb41463a7395c514d127d4f74fff12ef0137fff1b84b96c4" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.23.0" @@ -906,6 +979,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce04077ead78e39ae8610ad26216aed811996b043d47beed5090db674f9e9b5" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.6.0" @@ -948,6 +1040,15 @@ dependencies = [ "libc", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -969,6 +1070,16 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +[[package]] +name = "jpeg-decoder" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc797adac5f083b8ff0ca6f6294a999393d76e197c36488e2ef732c4715f6fa3" +dependencies = [ + "byteorder", + "rayon", +] + [[package]] name = "js-sys" version = "0.3.46" @@ -1027,6 +1138,15 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.3.9" @@ -1103,6 +1223,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5553f9f8090d7d74a2b321da9d7145d4636252e38a428386c6a920a9a937385a" +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + [[package]] name = "miniz_oxide" version = "0.4.3" @@ -1213,6 +1342,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -1272,6 +1423,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" version = "0.10.31" @@ -1305,6 +1462,31 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + [[package]] name = "percent-encoding" version = "1.0.1" @@ -1422,6 +1604,18 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1816,6 +2010,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1894,6 +2094,19 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +[[package]] +name = "sha2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpuid-bool", + "digest", + "opaque-debug", +] + [[package]] name = "siphasher" version = "0.3.3" @@ -1906,6 +2119,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "smallvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" + [[package]] name = "snekdown" version = "0.30.5" @@ -1922,6 +2141,7 @@ dependencies = [ "gh-emoji", "headless_chrome", "htmlescape", + "image", "indicatif", "lazy_static", "log 0.4.11", @@ -1930,12 +2150,14 @@ dependencies = [ "mime_guess", "minify", "notify", + "parking_lot", "platform-dirs", "rayon", "regex", "reqwest", "serde", "serde_derive", + "sha2", "structopt", "syntect", "toml", @@ -2096,6 +2318,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.3", + "weezl", +] + [[package]] name = "time" version = "0.1.44" @@ -2228,6 +2461,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + [[package]] name = "unicase" version = "1.4.2" @@ -2486,6 +2725,12 @@ dependencies = [ "url 1.7.2", ] +[[package]] +name = "weezl" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2bb9fc8309084dd7cd651336673844c1d47f8ef6d2091ec160b27f5c4aa277" + [[package]] name = "which" version = "2.0.1" diff --git a/Cargo.toml b/Cargo.toml index 149cd8d..3cdce71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ pdf = ["headless_chrome", "failure"] [dependencies] charred = "0.3.3" asciimath-rs = "0.5.7" -bibliographix = "0.5.0" +bibliographix = "0.6.0" crossbeam-utils = "0.7.2" structopt = "0.3.14" minify = "1.1.1" @@ -48,6 +48,9 @@ log = "0.4.11" env_logger = "0.7.1" indicatif = "0.15.0" platform-dirs = "0.2.0" +image = "0.23.12" +parking_lot = "0.11.1" +sha2 = "0.9.2" headless_chrome = {version = "0.9.0", optional = true} failure = {version = "0.1.8", optional = true} \ No newline at end of file diff --git a/README.md b/README.md index 8137383..b8b8004 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,26 @@ smart-arrows = true include-math-jax = true +### Image processing options ### + +# Force convert images to the specified format. +# Supported formats are png, jpeg, gif, bmp, (ico needs size <= 256), avif, pnm +# (default: keep original) +image-format = "jpg" + +# the max width for the images. +# if an image is larger than that it get's resized. +# (default: none) +image-max-width = 700 + +# the max width for the images. +# if an image is larger than that it get's resized. +# (default: none) +image-max-height = 800 + + ### PDF Options - needs the pdf feature enabled ### + # If the header and footer of the pdf should be displayed (default: true) pdf-display-header-footer = true diff --git a/src/elements/mod.rs b/src/elements/mod.rs index 3aae97c..12d4d8c 100644 --- a/src/elements/mod.rs +++ b/src/elements/mod.rs @@ -1,19 +1,26 @@ pub mod tokens; use crate::format::PlaceholderTemplate; +use crate::references::configuration::keys::{ + EMBED_EXTERNAL, IMAGE_FORMAT, IMAGE_MAX_HEIGHT, IMAGE_MAX_WIDTH, +}; use crate::references::configuration::{ConfigRefEntry, Configuration, Value}; use crate::references::glossary::{GlossaryManager, GlossaryReference}; use crate::references::placeholders::ProcessPlaceholders; use crate::references::templates::{Template, TemplateVariable}; use crate::utils::downloads::{DownloadManager, PendingDownload}; +use crate::utils::image_converting::{ImageConverter, PendingImage}; use asciimath_rs::elements::special::Expression; use bibliographix::bib_manager::BibManager; use bibliographix::bibliography::bibliography_entry::BibliographyEntryReference; use bibliographix::references::bib_reference::BibRefAnchor; +use image::ImageFormat; +use mime::Mime; +use parking_lot::Mutex; use std::collections::HashMap; use std::iter::FromIterator; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, RwLock}; pub const SECTION: &str = "section"; pub const PARAGRAPH: &str = "paragraph"; @@ -73,6 +80,7 @@ pub struct Document { pub config: Configuration, pub bibliography: BibManager, pub downloads: Arc>, + pub images: Arc>, pub stylesheets: Vec>>, pub glossary: Arc>, } @@ -236,7 +244,7 @@ pub struct Url { pub struct Image { pub(crate) url: Url, pub(crate) metadata: Option, - pub(crate) download: Arc>, + pub(crate) image_data: Arc>, } #[derive(Clone, Debug)] @@ -314,6 +322,7 @@ impl Document { bibliography: BibManager::new(), stylesheets: Vec::new(), downloads: Arc::new(Mutex::new(DownloadManager::new())), + images: Arc::new(Mutex::new(ImageConverter::new())), glossary: Arc::new(Mutex::new(GlossaryManager::new())), } } @@ -329,6 +338,7 @@ impl Document { bibliography: self.bibliography.create_child(), stylesheets: Vec::new(), downloads: Arc::clone(&self.downloads), + images: Arc::clone(&self.images), glossary: Arc::clone(&self.glossary), } } @@ -427,10 +437,59 @@ impl Document { if self.is_root { self.process_definitions(); self.bibliography.assign_entries_to_references(); - self.glossary.lock().unwrap().assign_entries_to_references(); + self.glossary.lock().assign_entries_to_references(); self.process_placeholders(); + self.process_media(); } } + + fn process_media(&self) { + let downloads = Arc::clone(&self.downloads); + if let Some(Value::Bool(embed)) = self + .config + .get_entry(EMBED_EXTERNAL) + .map(|e| e.get().clone()) + { + if embed { + downloads.lock().download_all(); + } + } else { + downloads.lock().download_all(); + } + if let Some(Value::String(s)) = self.config.get_entry(IMAGE_FORMAT).map(|e| e.get().clone()) + { + if let Some(format) = ImageFormat::from_extension(s) { + self.images.lock().set_target_format(format); + } + } + let mut image_width = -1; + let mut image_height = -1; + + if let Some(Value::Integer(i)) = self + .config + .get_entry(IMAGE_MAX_WIDTH) + .map(|v| v.get().clone()) + { + image_width = i; + image_height = i; + } + if let Some(Value::Integer(i)) = self + .config + .get_entry(IMAGE_MAX_HEIGHT) + .map(|v| v.get().clone()) + { + image_height = i; + if image_width < 0 { + image_width = i; + } + } + if image_width > 0 && image_height > 0 { + self.images + .lock() + .set_target_size((image_width as u32, image_height as u32)); + } + self.images.lock().convert_all(); + } } impl Section { @@ -703,10 +762,14 @@ impl Into> for InlineMetadata { impl Image { pub fn get_content(&self) -> Option> { let mut data = None; - std::mem::swap(&mut data, &mut self.download.lock().unwrap().data); + std::mem::swap(&mut data, &mut self.image_data.lock().data); data } + + pub fn get_mime_type(&self) -> Mime { + self.image_data.lock().mime.clone() + } } #[derive(Clone, Debug)] @@ -736,8 +799,8 @@ impl BibReference { } pub(crate) fn get_formatted(&self) -> String { - if let Some(entry) = &self.entry_anchor.lock().unwrap().entry { - let entry = entry.lock().unwrap(); + if let Some(entry) = &self.entry_anchor.lock().entry { + let entry = entry.lock(); if let Some(display) = &self.display { let display = display.read().unwrap(); diff --git a/src/format/html/to_html.rs b/src/format/html/to_html.rs index fc4ccf3..28a6f95 100644 --- a/src/format/html/to_html.rs +++ b/src/format/html/to_html.rs @@ -1,15 +1,13 @@ use crate::elements::*; use crate::format::html::html_writer::HTMLWriter; use crate::format::PlaceholderTemplate; -use crate::references::configuration::keys::{EMBED_EXTERNAL, INCLUDE_MATHJAX, META_LANG}; -use crate::references::configuration::Value; +use crate::references::configuration::keys::{INCLUDE_MATHJAX, META_LANG}; use crate::references::glossary::{GlossaryDisplay, GlossaryReference}; use crate::references::templates::{Template, TemplateVariable}; use asciimath_rs::format::mathml::ToMathML; use htmlescape::encode_attribute; use minify::html::minify; use std::io; -use std::sync::Arc; use syntect::highlighting::ThemeSet; use syntect::html::highlighted_html_for_string; use syntect::parsing::SyntaxSet; @@ -64,7 +62,7 @@ impl ToHtml for Inline { Inline::Math(m) => m.to_html(writer), Inline::LineBreak => writer.write("
".to_string()), Inline::CharacterCode(code) => code.to_html(writer), - Inline::GlossaryReference(gloss) => gloss.lock().unwrap().to_html(writer), + Inline::GlossaryReference(gloss) => gloss.lock().to_html(writer), Inline::Arrow(a) => a.to_html(writer), } } @@ -102,18 +100,6 @@ impl ToHtml for MetadataValue { impl ToHtml for Document { fn to_html(&self, writer: &mut HTMLWriter) -> io::Result<()> { - let downloads = Arc::clone(&self.downloads); - if let Some(Value::Bool(embed)) = self - .config - .get_entry(EMBED_EXTERNAL) - .map(|e| e.get().clone()) - { - if embed { - downloads.lock().unwrap().download_all(); - } - } else { - downloads.lock().unwrap().download_all(); - } let path = if let Some(path) = &self.path { format!("path=\"{}\"", encode_attribute(path.as_str())) } else { @@ -139,7 +125,7 @@ impl ToHtml for Document { writer.write("".to_string())?; for stylesheet in &self.stylesheets { - let mut stylesheet = stylesheet.lock().unwrap(); + let mut stylesheet = stylesheet.lock(); let data = std::mem::replace(&mut stylesheet.data, None); if let Some(data) = data { if self @@ -401,7 +387,7 @@ impl ToHtml for Image { let mut style = String::new(); let url = if let Some(content) = self.get_content() { - let mime_type = mime_guess::from_path(&self.url.url).first_or(mime::IMAGE_PNG); + let mime_type = self.get_mime_type(); format!( "data:{};base64,{}", mime_type.to_string(), @@ -668,7 +654,7 @@ impl ToHtml for Anchor { impl ToHtml for GlossaryReference { fn to_html(&self, writer: &mut HTMLWriter) -> io::Result<()> { if let Some(entry) = &self.entry { - let entry = entry.lock().unwrap(); + let entry = entry.lock(); writer.write("".to_string())?; diff --git a/src/main.rs b/src/main.rs index 3d49d4c..236a2de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -136,7 +136,7 @@ fn render(opt: &RenderOptions) -> Parser { ); let document = parser.parse(); - log::info!("Parsing took: {:?}", start.elapsed()); + log::info!("Parsing + Processing took: {:?}", start.elapsed()); let start_render = Instant::now(); let file = OpenOptions::new() @@ -149,8 +149,8 @@ fn render(opt: &RenderOptions) -> Parser { let writer = BufWriter::new(file); render_format(opt, document, writer); - log::info!("Rendering took: {:?}", start_render.elapsed()); - log::info!("Total: {:?}", start.elapsed()); + log::info!("Rendering took: {:?}", start_render.elapsed()); + log::info!("Total: {:?}", start.elapsed()); parser } diff --git a/src/parser/inline.rs b/src/parser/inline.rs index d4f3b29..9895c06 100644 --- a/src/parser/inline.rs +++ b/src/parser/inline.rs @@ -9,8 +9,10 @@ use crate::references::glossary::GlossaryReference; use crate::references::templates::{GetTemplateVariables, Template, TemplateVariable}; use crate::Parser; use bibliographix::references::bib_reference::BibRef; +use parking_lot::Mutex; use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; pub(crate) trait ParseInline { fn parse_surrounded(&mut self, surrounding: &char) -> ParseResult>; @@ -98,7 +100,7 @@ impl ParseInline for Parser { log::trace!("Inline::Striked"); Ok(Inline::Striked(striked)) } else if let Ok(gloss) = self.parse_glossary_reference() { - log::trace!("Inline::GlossaryReference {}", gloss.lock().unwrap().short); + log::trace!("Inline::GlossaryReference {}", gloss.lock().short); Ok(Inline::GlossaryReference(gloss)) } else if let Ok(superscript) = self.parse_superscript() { log::trace!("Inline::Superscript"); @@ -147,13 +149,12 @@ impl ParseInline for Parser { Ok(Image { url, metadata, - download: self + image_data: self .options .document - .downloads + .images .lock() - .unwrap() - .add_download(path), + .add_image(PathBuf::from(path)), }) } else { Err(self.ctm.rewind_with_error(start_index)) @@ -371,7 +372,6 @@ impl ParseInline for Parser { .bibliography .root_ref_anchor() .lock() - .unwrap() .insert(bib_ref); Ok(ref_entry) @@ -432,7 +432,6 @@ impl ParseInline for Parser { .document .glossary .lock() - .unwrap() .add_reference(reference)) } diff --git a/src/parser/line.rs b/src/parser/line.rs index aa9170b..0a50eb2 100644 --- a/src/parser/line.rs +++ b/src/parser/line.rs @@ -242,7 +242,6 @@ impl ParseLine for Parser { .bibliography .entry_dictionary() .lock() - .unwrap() .insert(entry); Ok(BibEntry { @@ -252,7 +251,6 @@ impl ParseLine for Parser { .bibliography .entry_dictionary() .lock() - .unwrap() .get(&key) .unwrap(), key, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c9d5d11..f4b1cf9 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -61,7 +61,7 @@ impl ParserOptions { /// If external sources should be cached when after downloaded pub fn use_cache(self, value: bool) -> Self { - self.document.downloads.lock().unwrap().use_cache = value; + self.document.downloads.lock().use_cache = value; self } @@ -209,7 +209,6 @@ impl Parser { .document .downloads .lock() - .unwrap() .add_download(path.to_str().unwrap().to_string()), ); @@ -236,7 +235,6 @@ impl Parser { .document .glossary .lock() - .unwrap() .assign_from_toml(value) .unwrap_or_else(|e| log::error!("{}", e)); diff --git a/src/references/bibliography.rs b/src/references/bibliography.rs index 923990b..c69cbc0 100644 --- a/src/references/bibliography.rs +++ b/src/references/bibliography.rs @@ -33,7 +33,6 @@ pub fn create_bib_list(entries: Vec) -> List { for entry in entries { entry .lock() - .unwrap() .raw_fields .insert("ord".to_string(), count.to_string()); list.add_item(get_item_for_entry(entry)); @@ -45,7 +44,7 @@ pub fn create_bib_list(entries: Vec) -> List { /// Returns the list item for a bib entry fn get_item_for_entry(entry: BibliographyEntryReference) -> ListItem { - let entry = entry.lock().unwrap(); + let entry = entry.lock(); match &entry.bib_type { BibliographyType::Article(a) => get_item_for_article(&*entry, a), diff --git a/src/references/configuration/keys.rs b/src/references/configuration/keys.rs index 7646ac5..cce5760 100644 --- a/src/references/configuration/keys.rs +++ b/src/references/configuration/keys.rs @@ -24,3 +24,8 @@ pub const PDF_MARGIN_RIGHT: &str = "pdf-margin-right"; pub const PDF_PAGE_HEIGHT: &str = "pdf-page-height"; pub const PDF_PAGE_WIDTH: &str = "pdf-page-width"; pub const PDF_PAGE_SCALE: &str = "pdf-page-scale"; + +// Image Options +pub const IMAGE_FORMAT: &str = "image-format"; +pub const IMAGE_MAX_WIDTH: &str = "image-max-width"; +pub const IMAGE_MAX_HEIGHT: &str = "image-max-height"; diff --git a/src/references/glossary.rs b/src/references/glossary.rs index 0751cbc..ba0238e 100644 --- a/src/references/glossary.rs +++ b/src/references/glossary.rs @@ -1,9 +1,10 @@ use crate::elements::{ Anchor, BoldText, Inline, ItalicText, Line, List, ListItem, PlainText, TextLine, }; +use parking_lot::Mutex; use std::cmp::Ordering; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use crate::bold_text; use crate::italic_text; @@ -110,11 +111,11 @@ impl GlossaryManager { /// Assignes entries to references pub fn assign_entries_to_references(&self) { for reference in &self.references { - let mut reference = reference.lock().unwrap(); + let mut reference = reference.lock(); if let Some(entry) = self.entries.get(&reference.short) { reference.entry = Some(Arc::clone(entry)); - let mut entry = entry.lock().unwrap(); + let mut entry = entry.lock(); if !entry.is_assigned { entry.is_assigned = true; @@ -130,13 +131,13 @@ impl GlossaryManager { let mut entries = self .entries .values() - .filter(|e| e.lock().unwrap().is_assigned) + .filter(|e| e.lock().is_assigned) .cloned() .collect::>>>(); entries.sort_by(|a, b| { - let a = a.lock().unwrap(); - let b = b.lock().unwrap(); + let a = a.lock(); + let b = b.lock(); if a.short > b.short { Ordering::Greater } else if a.short < b.short { @@ -146,7 +147,7 @@ impl GlossaryManager { } }); for entry in &entries { - let entry = entry.lock().unwrap(); + let entry = entry.lock(); let mut line = TextLine::new(); line.subtext.push(bold_text!(entry.short.clone())); line.subtext.push(plain_text!(" - ".to_string())); diff --git a/src/references/placeholders.rs b/src/references/placeholders.rs index 4aa19e7..f9c0be6 100644 --- a/src/references/placeholders.rs +++ b/src/references/placeholders.rs @@ -54,7 +54,7 @@ impl ProcessPlaceholders for Document { self.bibliography.get_entry_list_by_occurrence() )))), P_GLS => pholder.set_value(block!(Block::List( - self.glossary.lock().unwrap().create_glossary_list() + self.glossary.lock().create_glossary_list() ))), P_DATE => pholder.set_value(inline!(Inline::Plain(PlainText { value: get_date_string() diff --git a/src/utils/caching.rs b/src/utils/caching.rs index ce8fb3e..d9c6549 100644 --- a/src/utils/caching.rs +++ b/src/utils/caching.rs @@ -1,7 +1,6 @@ use platform_dirs::{AppDirs, AppUI}; -use std::collections::hash_map::DefaultHasher; +use sha2::Digest; use std::fs; -use std::hash::{Hash, Hasher}; use std::io; use std::path::PathBuf; @@ -23,9 +22,9 @@ impl CacheStorage { /// Returns the cache path for a given file pub fn get_file_path(&self, path: &PathBuf) -> PathBuf { - let mut hasher = DefaultHasher::new(); - path.hash(&mut hasher); - let mut file_name = PathBuf::from(format!("{:x}", hasher.finish())); + let mut hasher = sha2::Sha256::default(); + hasher.update(path.to_string_lossy().as_bytes()); + let mut file_name = PathBuf::from(format!("{:x}", hasher.finalize())); if let Some(extension) = path.extension() { file_name.set_extension(extension); diff --git a/src/utils/downloads.rs b/src/utils/downloads.rs index 4b0f535..c8d86f3 100644 --- a/src/utils/downloads.rs +++ b/src/utils/downloads.rs @@ -1,9 +1,10 @@ use crate::utils::caching::CacheStorage; use indicatif::{ProgressBar, ProgressStyle}; +use parking_lot::Mutex; use rayon::prelude::*; use std::fs::read; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; /// A manager for downloading urls in parallel #[derive(Clone, Debug)] @@ -35,7 +36,7 @@ impl DownloadManager { /// Downloads all download entries pub fn download_all(&self) { let pb = Arc::new(Mutex::new(ProgressBar::new(self.downloads.len() as u64))); - pb.lock().unwrap().set_style( + pb.lock().set_style( ProgressStyle::default_bar() .template("Fetching Embeds: [{bar:40.cyan/blue}]") .progress_chars("=> "), @@ -43,10 +44,10 @@ impl DownloadManager { let pb_cloned = Arc::clone(&pb); self.downloads.par_iter().for_each_with(pb_cloned, |pb, d| { - d.lock().unwrap().download(); - pb.lock().unwrap().inc(1); + d.lock().download(); + pb.lock().inc(1); }); - pb.lock().unwrap().finish_and_clear(); + pb.lock().finish_and_clear(); } } @@ -116,10 +117,14 @@ impl PendingDownload { /// Downloads the content from the given url fn download_content(&self) -> Option> { - reqwest::blocking::get(&self.path) - .ok() - .map(|c| c.bytes()) - .and_then(|b| b.ok()) - .map(|b| b.to_vec()) + download_path(self.path.clone()) } } + +pub fn download_path(path: String) -> Option> { + reqwest::blocking::get(&path) + .ok() + .map(|c| c.bytes()) + .and_then(|b| b.ok()) + .map(|b| b.to_vec()) +} diff --git a/src/utils/image_converting.rs b/src/utils/image_converting.rs new file mode 100644 index 0000000..d0d5aab --- /dev/null +++ b/src/utils/image_converting.rs @@ -0,0 +1,176 @@ +use crate::utils::caching::CacheStorage; +use crate::utils::downloads::download_path; +use image::imageops::FilterType; +use image::io::Reader as ImageReader; +use image::{GenericImageView, ImageFormat, ImageResult}; +use mime::Mime; +use parking_lot::Mutex; +use rayon::prelude::*; +use std::io; +use std::io::Cursor; +use std::path::PathBuf; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct ImageConverter { + images: Vec>>, + target_format: Option, + target_size: Option<(u32, u32)>, +} + +impl ImageConverter { + pub fn new() -> Self { + Self { + images: Vec::new(), + target_format: None, + target_size: None, + } + } + + pub fn set_target_size(&mut self, target_size: (u32, u32)) { + self.target_size = Some(target_size) + } + + pub fn set_target_format(&mut self, target_format: ImageFormat) { + self.target_format = Some(target_format); + } + + /// Adds an image to convert + pub fn add_image(&mut self, path: PathBuf) -> Arc> { + let image = Arc::new(Mutex::new(PendingImage::new(path))); + self.images.push(image.clone()); + + image + } + + /// Converts all images + pub fn convert_all(&mut self) { + self.images.par_iter().for_each(|image| { + let mut image = image.lock(); + if let Err(e) = image.convert(self.target_format.clone(), self.target_size.clone()) { + log::error!("Failed to embed image {:?}: {}", image.path, e) + } + }); + } +} + +#[derive(Clone, Debug)] +pub struct PendingImage { + pub path: PathBuf, + pub data: Option>, + cache: CacheStorage, + pub mime: Mime, +} + +impl PendingImage { + pub fn new(path: PathBuf) -> Self { + let mime = get_mime(&path); + + Self { + path, + data: None, + cache: CacheStorage::new(), + mime, + } + } + + /// Converts the image to the specified target format (specified by target_extension) + pub fn convert( + &mut self, + target_format: Option, + target_size: Option<(u32, u32)>, + ) -> ImageResult<()> { + let format = target_format + .or_else(|| { + self.path + .extension() + .and_then(|extension| ImageFormat::from_extension(extension)) + }) + .unwrap_or(ImageFormat::Png); + let output_path = self.get_output_path(format, target_size); + self.mime = get_mime(&output_path); + + if self.cache.has_file(&output_path) { + self.data = Some(self.cache.read(&output_path)?) + } else { + self.convert_image(format, target_size)?; + + if let Some(data) = &self.data { + self.cache.write(&output_path, data)?; + } + } + + Ok(()) + } + + /// Converts the image + fn convert_image( + &mut self, + format: ImageFormat, + target_size: Option<(u32, u32)>, + ) -> ImageResult<()> { + let mut image = ImageReader::open(self.get_path()?)?.decode()?; + + if let Some((width, height)) = target_size { + let dimensions = image.dimensions(); + + if dimensions.0 > width || dimensions.1 > height { + image = image.resize(width, height, FilterType::Lanczos3); + } + } + let data = Vec::new(); + let mut writer = Cursor::new(data); + + image.write_to(&mut writer, format)?; + self.data = Some(writer.into_inner()); + + Ok(()) + } + + /// Returns the path of the file + fn get_path(&self) -> io::Result { + if !self.path.exists() { + if self.cache.has_file(&self.path) { + return Ok(self.cache.get_file_path(&self.path)); + } + if let Some(data) = download_path(self.path.to_string_lossy().to_string()) { + self.cache.write(&self.path, data)?; + return Ok(self.cache.get_file_path(&self.path)); + } + } + Ok(self.path.clone()) + } + + /// Returns the output file name after converting the image + fn get_output_path( + &self, + target_format: ImageFormat, + target_size: Option<(u32, u32)>, + ) -> PathBuf { + let mut path = self.path.clone(); + let mut file_name = path.file_stem().unwrap().to_string_lossy().to_string(); + let extension = target_format.extensions_str()[0]; + let type_name = format!("{:?}", target_format); + + if let Some(target_size) = target_size { + file_name += &*format!("-{}-{}", target_size.0, target_size.1); + } + file_name += format!("-{}-converted", type_name).as_str(); + path.set_file_name(file_name); + path.set_extension(extension); + + path + } +} + +fn get_mime(path: &PathBuf) -> Mime { + let mime = mime_guess::from_ext( + path.clone() + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("png"), + ) + .first() + .unwrap_or(mime::IMAGE_PNG); + mime +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index d1229a1..7beaff7 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,4 +1,5 @@ pub mod caching; pub mod downloads; +pub mod image_converting; pub mod macros; pub mod parsing;