From 53c4818f9dbc75de1dd7b7c6b2de8e1f55451a7d Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 16 Dec 2020 11:55:56 +0100 Subject: [PATCH 1/6] Change input syntax and add cache storage handler - input syntax changed to - Added clear-cache subcommand - CacheStorage now handles the caching of files Signed-off-by: trivernis --- src/format/chromium_pdf/mod.rs | 5 +-- src/main.rs | 66 ++++++++++++++++++++-------------- src/utils/caching.rs | 63 ++++++++++++++++++++++++++++++++ src/utils/downloads.rs | 43 ++++++---------------- src/utils/mod.rs | 1 + 5 files changed, 118 insertions(+), 60 deletions(-) create mode 100644 src/utils/caching.rs diff --git a/src/format/chromium_pdf/mod.rs b/src/format/chromium_pdf/mod.rs index 77f39b5..748d8a1 100644 --- a/src/format/chromium_pdf/mod.rs +++ b/src/format/chromium_pdf/mod.rs @@ -8,7 +8,7 @@ use crate::references::configuration::keys::{ PDF_PAGE_SCALE, PDF_PAGE_WIDTH, }; use crate::references::configuration::Configuration; -use crate::utils::downloads::get_cached_path; +use crate::utils::caching::CacheStorage; use headless_chrome::protocol::page::PrintToPdfOptions; use headless_chrome::{Browser, LaunchOptionsBuilder, Tab}; use std::fs; @@ -22,8 +22,9 @@ pub mod result; /// Renders the document to pdf and returns the resulting bytes pub fn render_to_pdf(document: Document) -> PdfRenderingResult> { + let cache = CacheStorage::new(); let mut file_path = PathBuf::from(format!("tmp-document.html")); - file_path = get_cached_path(file_path).with_extension("html"); + file_path = cache.get_file_path(&file_path); let mut mathjax = false; if let Some(entry) = document.config.get_entry(INCLUDE_MATHJAX) { diff --git a/src/main.rs b/src/main.rs index 0b3db3b..3d49d4c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,16 +6,38 @@ use snekdown::elements::Document; use snekdown::format::html::html_writer::HTMLWriter; use snekdown::format::html::to_html::ToHtml; use snekdown::parser::ParserOptions; +use snekdown::utils::caching::CacheStorage; use snekdown::Parser; use std::fs::{File, OpenOptions}; use std::io::{BufWriter, Write}; use std::path::PathBuf; +use std::process::exit; use std::sync::mpsc::channel; use std::time::{Duration, Instant}; use structopt::StructOpt; #[derive(StructOpt, Debug)] struct Opt { + #[structopt(subcommand)] + sub_command: SubCommand, +} + +#[derive(StructOpt, Debug)] +#[structopt()] +enum SubCommand { + /// Watch the document and its imports and render on change. + Watch(RenderOptions), + + /// Parse and render the document. + Render(RenderOptions), + + /// Clears the cache directory + ClearCache, +} + +#[derive(StructOpt, Debug)] +#[structopt()] +struct RenderOptions { /// Path to the input file #[structopt(parse(from_os_str))] input: PathBuf, @@ -31,19 +53,6 @@ struct Opt { /// Don't use the cache #[structopt(long)] no_cache: bool, - - #[structopt(subcommand)] - sub_command: Option, -} - -#[derive(StructOpt, Debug)] -#[structopt()] -enum SubCommand { - /// Watch the document and its imports and render on change. - Watch, - - /// Default. Parse and render the document. - Render, } fn main() { @@ -68,19 +77,16 @@ fn main() { ) }) .init(); - if !opt.input.exists() { - log::error!( - "The input file {} could not be found", - opt.input.to_str().unwrap() - ); - return; - } match &opt.sub_command { - Some(SubCommand::Render) | None => { + SubCommand::Render(opt) => { let _ = render(&opt); } - Some(SubCommand::Watch) => watch(&opt), + SubCommand::Watch(opt) => watch(&opt), + SubCommand::ClearCache => { + let cache = CacheStorage::new(); + cache.clear().expect("Failed to clear cache"); + } }; } @@ -95,7 +101,7 @@ fn get_level_style(level: Level) -> colored::Color { } /// Watches a file with all of its imports and renders on change -fn watch(opt: &Opt) { +fn watch(opt: &RenderOptions) { let parser = render(opt); let (tx, rx) = channel(); let mut watcher = watcher(tx, Duration::from_millis(250)).unwrap(); @@ -112,7 +118,15 @@ fn watch(opt: &Opt) { } /// Renders the document to the output path -fn render(opt: &Opt) -> Parser { +fn render(opt: &RenderOptions) -> Parser { + if !opt.input.exists() { + log::error!( + "The input file {} could not be found", + opt.input.to_str().unwrap() + ); + + exit(1) + } let start = Instant::now(); let mut parser = Parser::with_defaults( @@ -142,7 +156,7 @@ fn render(opt: &Opt) -> Parser { } #[cfg(not(feature = "pdf"))] -fn render_format(opt: &Opt, document: Document, writer: BufWriter) { +fn render_format(opt: &RenderOptions, document: Document, writer: BufWriter) { match opt.format.as_str() { "html" => render_html(document, writer), _ => log::error!("Unknown format {}", opt.format), @@ -150,7 +164,7 @@ fn render_format(opt: &Opt, document: Document, writer: BufWriter) { } #[cfg(feature = "pdf")] -fn render_format(opt: &Opt, document: Document, writer: BufWriter) { +fn render_format(opt: &RenderOptions, document: Document, writer: BufWriter) { match opt.format.as_str() { "html" => render_html(document, writer), "pdf" => render_pdf(document, writer), diff --git a/src/utils/caching.rs b/src/utils/caching.rs new file mode 100644 index 0000000..ce8fb3e --- /dev/null +++ b/src/utils/caching.rs @@ -0,0 +1,63 @@ +use platform_dirs::{AppDirs, AppUI}; +use std::collections::hash_map::DefaultHasher; +use std::fs; +use std::hash::{Hash, Hasher}; +use std::io; +use std::path::PathBuf; + +#[derive(Clone, Debug)] +pub struct CacheStorage { + location: PathBuf, +} + +impl CacheStorage { + pub fn new() -> Self { + lazy_static::lazy_static! { + static ref APP_DIRS: AppDirs = AppDirs::new(Some("snekdown"), AppUI::CommandLine).unwrap(); + } + + Self { + location: APP_DIRS.cache_dir.clone(), + } + } + + /// Returns the cache path for a given file + pub fn get_file_path(&self, path: &PathBuf) -> PathBuf { + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + let mut file_name = PathBuf::from(format!("{:x}", hasher.finish())); + + if let Some(extension) = path.extension() { + file_name.set_extension(extension); + } + + return self.location.join(PathBuf::from(file_name)); + } + + /// Returns if the given file exists in the cache + pub fn has_file(&self, path: &PathBuf) -> bool { + let cache_path = self.get_file_path(path); + + cache_path.exists() + } + + /// Writes into the corresponding cache file + pub fn read(&self, path: &PathBuf) -> io::Result> { + let cache_path = self.get_file_path(path); + + fs::read(cache_path) + } + + /// Reads the corresponding cache file + pub fn write>(&self, path: &PathBuf, contents: R) -> io::Result<()> { + let cache_path = self.get_file_path(path); + + fs::write(cache_path, contents) + } + + /// Clears the cache directory by deleting and recreating it + pub fn clear(&self) -> io::Result<()> { + fs::remove_dir_all(&self.location)?; + fs::create_dir(&self.location) + } +} diff --git a/src/utils/downloads.rs b/src/utils/downloads.rs index 118f26d..4b0f535 100644 --- a/src/utils/downloads.rs +++ b/src/utils/downloads.rs @@ -1,10 +1,7 @@ +use crate::utils::caching::CacheStorage; use indicatif::{ProgressBar, ProgressStyle}; -use platform_dirs::{AppDirs, AppUI}; use rayon::prelude::*; -use std::collections::hash_map::DefaultHasher; -use std::fs; use std::fs::read; -use std::hash::{Hash, Hasher}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -60,6 +57,7 @@ pub struct PendingDownload { pub(crate) path: String, pub(crate) data: Option>, pub(crate) use_cache: bool, + cache: CacheStorage, } impl PendingDownload { @@ -68,6 +66,7 @@ impl PendingDownload { path, data: None, use_cache: true, + cache: CacheStorage::new(), } } @@ -98,22 +97,18 @@ impl PendingDownload { /// Stores the data to a cache file to retrieve it later fn store_to_cache(&self, data: &Vec) { if self.use_cache { - let cache_file = get_cached_path(PathBuf::from(&self.path)); - log::debug!("Writing to cache {} -> {:?}", self.path, cache_file); - fs::write(&cache_file, data.clone()).unwrap_or_else(|_| { - log::warn!( - "Failed to write file to cache: {} -> {:?}", - self.path, - cache_file - ) - }); + let path = PathBuf::from(&self.path); + self.cache + .write(&path, data.clone()) + .unwrap_or_else(|_| log::warn!("Failed to write file to cache: {}", self.path)); } } fn read_from_cache(&self) -> Option> { - let cache_path = get_cached_path(PathBuf::from(&self.path)); - if cache_path.exists() && self.use_cache { - read(cache_path).ok() + let path = PathBuf::from(&self.path); + + if self.cache.has_file(&path) && self.use_cache { + self.cache.read(&path).ok() } else { None } @@ -128,19 +123,3 @@ impl PendingDownload { .map(|b| b.to_vec()) } } - -pub fn get_cached_path(path: PathBuf) -> PathBuf { - lazy_static::lazy_static! { - static ref APP_DIRS: AppDirs = AppDirs::new(Some("snekdown"), AppUI::CommandLine).unwrap(); - } - let mut hasher = DefaultHasher::new(); - path.hash(&mut hasher); - let file_name = PathBuf::from(format!("{:x}", hasher.finish())); - - if !APP_DIRS.cache_dir.is_dir() { - fs::create_dir(&APP_DIRS.cache_dir) - .unwrap_or_else(|_| log::warn!("Failed to create cache dir {:?}", APP_DIRS.cache_dir)) - } - - APP_DIRS.cache_dir.join(file_name) -} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 20780f3..d1229a1 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,3 +1,4 @@ +pub mod caching; pub mod downloads; pub mod macros; pub mod parsing; From 3acfe9c6e2a313a93211e06e0253369141cc0a4a Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 16 Dec 2020 11:58:14 +0100 Subject: [PATCH 2/6] Update Readme Signed-off-by: trivernis --- README.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 919831a..8137383 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,29 @@ cargo install snekdown --features pdf ## Usage ``` +snekdown 0.30.5 + USAGE: - snekdown [FLAGS] [OPTIONS] [SUBCOMMAND] + snekdown + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +SUBCOMMANDS: + clear-cache Clears the cache directory + help Prints this message or the help of the given subcommand(s) + render Parse and render the document + watch Watch the document and its imports and render on change +``` + +### Rendering / Watching + +``` +Parse and render the document + +USAGE: + snekdown render [FLAGS] [OPTIONS] FLAGS: -h, --help Prints help information @@ -35,11 +56,6 @@ OPTIONS: ARGS: Path to the input file Path for the output file - -SUBCOMMANDS: - help Prints this message or the help of the given subcommand(s) - render Default. Parse and render the document - watch Watch the document and its imports and render on change ``` ## Syntax From 63ea60b10afc3d53dd2ef42f8f1c8339284af153 Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 16 Dec 2020 16:14:20 +0100 Subject: [PATCH 3/6] Add conversion of images when configured Signed-off-by: trivernis --- Cargo.lock | 253 ++++++++++++++++++++++++++- Cargo.toml | 5 +- README.md | 19 ++ src/elements/mod.rs | 75 +++++++- src/format/html/to_html.rs | 24 +-- src/main.rs | 6 +- src/parser/inline.rs | 15 +- src/parser/line.rs | 2 - src/parser/mod.rs | 4 +- src/references/bibliography.rs | 3 +- src/references/configuration/keys.rs | 5 + src/references/glossary.rs | 15 +- src/references/placeholders.rs | 2 +- src/utils/caching.rs | 9 +- src/utils/downloads.rs | 25 +-- src/utils/image_converting.rs | 176 +++++++++++++++++++ src/utils/mod.rs | 1 + 17 files changed, 568 insertions(+), 71 deletions(-) create mode 100644 src/utils/image_converting.rs diff --git a/Cargo.lock b/Cargo.lock index e412a83..edf87b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aho-corasick" version = "0.7.15" @@ -89,7 +95,7 @@ dependencies = [ "addr2line", "cfg-if 1.0.0", "libc", - "miniz_oxide", + "miniz_oxide 0.4.3", "object", "rustc-demangle", ] @@ -127,12 +133,13 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bibliographix" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39034545c510b822e3e5bd76147f869bae617d52961add6313ad0696084585c1" +checksum = "bef9342b1214c0ff300bb812af4c9a35e0ec1351037a2a3e8595f24483a953d3" dependencies = [ "chrono", "chrono-english", + "parking_lot", "toml", ] @@ -163,12 +170,27 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +[[package]] +name = "bytemuck" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41aa2ec95ca3b5c54cf73c91acf06d24f4495d5f1b1c12506ae3483d646177ac" + [[package]] name = "byteorder" version = "1.3.4" @@ -253,6 +275,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colored" version = "1.9.3" @@ -318,6 +346,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + [[package]] name = "crc32fast" version = "1.2.1" @@ -419,6 +453,16 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", +] + [[package]] name = "derive_builder" version = "0.7.2" @@ -444,6 +488,15 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "dirs" version = "2.0.2" @@ -555,7 +608,7 @@ dependencies = [ "cfg-if 1.0.0", "crc32fast", "libc", - "miniz_oxide", + "miniz_oxide 0.4.3", ] [[package]] @@ -681,6 +734,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check 0.9.2", +] + [[package]] name = "getrandom" version = "0.1.15" @@ -702,6 +765,16 @@ dependencies = [ "regex", ] +[[package]] +name = "gif" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02efba560f227847cb41463a7395c514d127d4f74fff12ef0137fff1b84b96c4" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.23.0" @@ -906,6 +979,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce04077ead78e39ae8610ad26216aed811996b043d47beed5090db674f9e9b5" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.6.0" @@ -948,6 +1040,15 @@ dependencies = [ "libc", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -969,6 +1070,16 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +[[package]] +name = "jpeg-decoder" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc797adac5f083b8ff0ca6f6294a999393d76e197c36488e2ef732c4715f6fa3" +dependencies = [ + "byteorder", + "rayon", +] + [[package]] name = "js-sys" version = "0.3.46" @@ -1027,6 +1138,15 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.3.9" @@ -1103,6 +1223,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5553f9f8090d7d74a2b321da9d7145d4636252e38a428386c6a920a9a937385a" +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + [[package]] name = "miniz_oxide" version = "0.4.3" @@ -1213,6 +1342,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -1272,6 +1423,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" version = "0.10.31" @@ -1305,6 +1462,31 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + [[package]] name = "percent-encoding" version = "1.0.1" @@ -1422,6 +1604,18 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1816,6 +2010,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1894,6 +2094,19 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +[[package]] +name = "sha2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpuid-bool", + "digest", + "opaque-debug", +] + [[package]] name = "siphasher" version = "0.3.3" @@ -1906,6 +2119,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "smallvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" + [[package]] name = "snekdown" version = "0.30.5" @@ -1922,6 +2141,7 @@ dependencies = [ "gh-emoji", "headless_chrome", "htmlescape", + "image", "indicatif", "lazy_static", "log 0.4.11", @@ -1930,12 +2150,14 @@ dependencies = [ "mime_guess", "minify", "notify", + "parking_lot", "platform-dirs", "rayon", "regex", "reqwest", "serde", "serde_derive", + "sha2", "structopt", "syntect", "toml", @@ -2096,6 +2318,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.3", + "weezl", +] + [[package]] name = "time" version = "0.1.44" @@ -2228,6 +2461,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + [[package]] name = "unicase" version = "1.4.2" @@ -2486,6 +2725,12 @@ dependencies = [ "url 1.7.2", ] +[[package]] +name = "weezl" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2bb9fc8309084dd7cd651336673844c1d47f8ef6d2091ec160b27f5c4aa277" + [[package]] name = "which" version = "2.0.1" diff --git a/Cargo.toml b/Cargo.toml index 149cd8d..3cdce71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ pdf = ["headless_chrome", "failure"] [dependencies] charred = "0.3.3" asciimath-rs = "0.5.7" -bibliographix = "0.5.0" +bibliographix = "0.6.0" crossbeam-utils = "0.7.2" structopt = "0.3.14" minify = "1.1.1" @@ -48,6 +48,9 @@ log = "0.4.11" env_logger = "0.7.1" indicatif = "0.15.0" platform-dirs = "0.2.0" +image = "0.23.12" +parking_lot = "0.11.1" +sha2 = "0.9.2" headless_chrome = {version = "0.9.0", optional = true} failure = {version = "0.1.8", optional = true} \ No newline at end of file diff --git a/README.md b/README.md index 8137383..b8b8004 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,26 @@ smart-arrows = true include-math-jax = true +### Image processing options ### + +# Force convert images to the specified format. +# Supported formats are png, jpeg, gif, bmp, (ico needs size <= 256), avif, pnm +# (default: keep original) +image-format = "jpg" + +# the max width for the images. +# if an image is larger than that it get's resized. +# (default: none) +image-max-width = 700 + +# the max width for the images. +# if an image is larger than that it get's resized. +# (default: none) +image-max-height = 800 + + ### PDF Options - needs the pdf feature enabled ### + # If the header and footer of the pdf should be displayed (default: true) pdf-display-header-footer = true diff --git a/src/elements/mod.rs b/src/elements/mod.rs index 3aae97c..12d4d8c 100644 --- a/src/elements/mod.rs +++ b/src/elements/mod.rs @@ -1,19 +1,26 @@ pub mod tokens; use crate::format::PlaceholderTemplate; +use crate::references::configuration::keys::{ + EMBED_EXTERNAL, IMAGE_FORMAT, IMAGE_MAX_HEIGHT, IMAGE_MAX_WIDTH, +}; use crate::references::configuration::{ConfigRefEntry, Configuration, Value}; use crate::references::glossary::{GlossaryManager, GlossaryReference}; use crate::references::placeholders::ProcessPlaceholders; use crate::references::templates::{Template, TemplateVariable}; use crate::utils::downloads::{DownloadManager, PendingDownload}; +use crate::utils::image_converting::{ImageConverter, PendingImage}; use asciimath_rs::elements::special::Expression; use bibliographix::bib_manager::BibManager; use bibliographix::bibliography::bibliography_entry::BibliographyEntryReference; use bibliographix::references::bib_reference::BibRefAnchor; +use image::ImageFormat; +use mime::Mime; +use parking_lot::Mutex; use std::collections::HashMap; use std::iter::FromIterator; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, RwLock}; pub const SECTION: &str = "section"; pub const PARAGRAPH: &str = "paragraph"; @@ -73,6 +80,7 @@ pub struct Document { pub config: Configuration, pub bibliography: BibManager, pub downloads: Arc>, + pub images: Arc>, pub stylesheets: Vec>>, pub glossary: Arc>, } @@ -236,7 +244,7 @@ pub struct Url { pub struct Image { pub(crate) url: Url, pub(crate) metadata: Option, - pub(crate) download: Arc>, + pub(crate) image_data: Arc>, } #[derive(Clone, Debug)] @@ -314,6 +322,7 @@ impl Document { bibliography: BibManager::new(), stylesheets: Vec::new(), downloads: Arc::new(Mutex::new(DownloadManager::new())), + images: Arc::new(Mutex::new(ImageConverter::new())), glossary: Arc::new(Mutex::new(GlossaryManager::new())), } } @@ -329,6 +338,7 @@ impl Document { bibliography: self.bibliography.create_child(), stylesheets: Vec::new(), downloads: Arc::clone(&self.downloads), + images: Arc::clone(&self.images), glossary: Arc::clone(&self.glossary), } } @@ -427,10 +437,59 @@ impl Document { if self.is_root { self.process_definitions(); self.bibliography.assign_entries_to_references(); - self.glossary.lock().unwrap().assign_entries_to_references(); + self.glossary.lock().assign_entries_to_references(); self.process_placeholders(); + self.process_media(); } } + + fn process_media(&self) { + let downloads = Arc::clone(&self.downloads); + if let Some(Value::Bool(embed)) = self + .config + .get_entry(EMBED_EXTERNAL) + .map(|e| e.get().clone()) + { + if embed { + downloads.lock().download_all(); + } + } else { + downloads.lock().download_all(); + } + if let Some(Value::String(s)) = self.config.get_entry(IMAGE_FORMAT).map(|e| e.get().clone()) + { + if let Some(format) = ImageFormat::from_extension(s) { + self.images.lock().set_target_format(format); + } + } + let mut image_width = -1; + let mut image_height = -1; + + if let Some(Value::Integer(i)) = self + .config + .get_entry(IMAGE_MAX_WIDTH) + .map(|v| v.get().clone()) + { + image_width = i; + image_height = i; + } + if let Some(Value::Integer(i)) = self + .config + .get_entry(IMAGE_MAX_HEIGHT) + .map(|v| v.get().clone()) + { + image_height = i; + if image_width < 0 { + image_width = i; + } + } + if image_width > 0 && image_height > 0 { + self.images + .lock() + .set_target_size((image_width as u32, image_height as u32)); + } + self.images.lock().convert_all(); + } } impl Section { @@ -703,10 +762,14 @@ impl Into> for InlineMetadata { impl Image { pub fn get_content(&self) -> Option> { let mut data = None; - std::mem::swap(&mut data, &mut self.download.lock().unwrap().data); + std::mem::swap(&mut data, &mut self.image_data.lock().data); data } + + pub fn get_mime_type(&self) -> Mime { + self.image_data.lock().mime.clone() + } } #[derive(Clone, Debug)] @@ -736,8 +799,8 @@ impl BibReference { } pub(crate) fn get_formatted(&self) -> String { - if let Some(entry) = &self.entry_anchor.lock().unwrap().entry { - let entry = entry.lock().unwrap(); + if let Some(entry) = &self.entry_anchor.lock().entry { + let entry = entry.lock(); if let Some(display) = &self.display { let display = display.read().unwrap(); diff --git a/src/format/html/to_html.rs b/src/format/html/to_html.rs index fc4ccf3..28a6f95 100644 --- a/src/format/html/to_html.rs +++ b/src/format/html/to_html.rs @@ -1,15 +1,13 @@ use crate::elements::*; use crate::format::html::html_writer::HTMLWriter; use crate::format::PlaceholderTemplate; -use crate::references::configuration::keys::{EMBED_EXTERNAL, INCLUDE_MATHJAX, META_LANG}; -use crate::references::configuration::Value; +use crate::references::configuration::keys::{INCLUDE_MATHJAX, META_LANG}; use crate::references::glossary::{GlossaryDisplay, GlossaryReference}; use crate::references::templates::{Template, TemplateVariable}; use asciimath_rs::format::mathml::ToMathML; use htmlescape::encode_attribute; use minify::html::minify; use std::io; -use std::sync::Arc; use syntect::highlighting::ThemeSet; use syntect::html::highlighted_html_for_string; use syntect::parsing::SyntaxSet; @@ -64,7 +62,7 @@ impl ToHtml for Inline { Inline::Math(m) => m.to_html(writer), Inline::LineBreak => writer.write("
".to_string()), Inline::CharacterCode(code) => code.to_html(writer), - Inline::GlossaryReference(gloss) => gloss.lock().unwrap().to_html(writer), + Inline::GlossaryReference(gloss) => gloss.lock().to_html(writer), Inline::Arrow(a) => a.to_html(writer), } } @@ -102,18 +100,6 @@ impl ToHtml for MetadataValue { impl ToHtml for Document { fn to_html(&self, writer: &mut HTMLWriter) -> io::Result<()> { - let downloads = Arc::clone(&self.downloads); - if let Some(Value::Bool(embed)) = self - .config - .get_entry(EMBED_EXTERNAL) - .map(|e| e.get().clone()) - { - if embed { - downloads.lock().unwrap().download_all(); - } - } else { - downloads.lock().unwrap().download_all(); - } let path = if let Some(path) = &self.path { format!("path=\"{}\"", encode_attribute(path.as_str())) } else { @@ -139,7 +125,7 @@ impl ToHtml for Document { writer.write("".to_string())?; for stylesheet in &self.stylesheets { - let mut stylesheet = stylesheet.lock().unwrap(); + let mut stylesheet = stylesheet.lock(); let data = std::mem::replace(&mut stylesheet.data, None); if let Some(data) = data { if self @@ -401,7 +387,7 @@ impl ToHtml for Image { let mut style = String::new(); let url = if let Some(content) = self.get_content() { - let mime_type = mime_guess::from_path(&self.url.url).first_or(mime::IMAGE_PNG); + let mime_type = self.get_mime_type(); format!( "data:{};base64,{}", mime_type.to_string(), @@ -668,7 +654,7 @@ impl ToHtml for Anchor { impl ToHtml for GlossaryReference { fn to_html(&self, writer: &mut HTMLWriter) -> io::Result<()> { if let Some(entry) = &self.entry { - let entry = entry.lock().unwrap(); + let entry = entry.lock(); writer.write("".to_string())?; diff --git a/src/main.rs b/src/main.rs index 3d49d4c..236a2de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -136,7 +136,7 @@ fn render(opt: &RenderOptions) -> Parser { ); let document = parser.parse(); - log::info!("Parsing took: {:?}", start.elapsed()); + log::info!("Parsing + Processing took: {:?}", start.elapsed()); let start_render = Instant::now(); let file = OpenOptions::new() @@ -149,8 +149,8 @@ fn render(opt: &RenderOptions) -> Parser { let writer = BufWriter::new(file); render_format(opt, document, writer); - log::info!("Rendering took: {:?}", start_render.elapsed()); - log::info!("Total: {:?}", start.elapsed()); + log::info!("Rendering took: {:?}", start_render.elapsed()); + log::info!("Total: {:?}", start.elapsed()); parser } diff --git a/src/parser/inline.rs b/src/parser/inline.rs index d4f3b29..9895c06 100644 --- a/src/parser/inline.rs +++ b/src/parser/inline.rs @@ -9,8 +9,10 @@ use crate::references::glossary::GlossaryReference; use crate::references::templates::{GetTemplateVariables, Template, TemplateVariable}; use crate::Parser; use bibliographix::references::bib_reference::BibRef; +use parking_lot::Mutex; use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; pub(crate) trait ParseInline { fn parse_surrounded(&mut self, surrounding: &char) -> ParseResult>; @@ -98,7 +100,7 @@ impl ParseInline for Parser { log::trace!("Inline::Striked"); Ok(Inline::Striked(striked)) } else if let Ok(gloss) = self.parse_glossary_reference() { - log::trace!("Inline::GlossaryReference {}", gloss.lock().unwrap().short); + log::trace!("Inline::GlossaryReference {}", gloss.lock().short); Ok(Inline::GlossaryReference(gloss)) } else if let Ok(superscript) = self.parse_superscript() { log::trace!("Inline::Superscript"); @@ -147,13 +149,12 @@ impl ParseInline for Parser { Ok(Image { url, metadata, - download: self + image_data: self .options .document - .downloads + .images .lock() - .unwrap() - .add_download(path), + .add_image(PathBuf::from(path)), }) } else { Err(self.ctm.rewind_with_error(start_index)) @@ -371,7 +372,6 @@ impl ParseInline for Parser { .bibliography .root_ref_anchor() .lock() - .unwrap() .insert(bib_ref); Ok(ref_entry) @@ -432,7 +432,6 @@ impl ParseInline for Parser { .document .glossary .lock() - .unwrap() .add_reference(reference)) } diff --git a/src/parser/line.rs b/src/parser/line.rs index aa9170b..0a50eb2 100644 --- a/src/parser/line.rs +++ b/src/parser/line.rs @@ -242,7 +242,6 @@ impl ParseLine for Parser { .bibliography .entry_dictionary() .lock() - .unwrap() .insert(entry); Ok(BibEntry { @@ -252,7 +251,6 @@ impl ParseLine for Parser { .bibliography .entry_dictionary() .lock() - .unwrap() .get(&key) .unwrap(), key, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c9d5d11..f4b1cf9 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -61,7 +61,7 @@ impl ParserOptions { /// If external sources should be cached when after downloaded pub fn use_cache(self, value: bool) -> Self { - self.document.downloads.lock().unwrap().use_cache = value; + self.document.downloads.lock().use_cache = value; self } @@ -209,7 +209,6 @@ impl Parser { .document .downloads .lock() - .unwrap() .add_download(path.to_str().unwrap().to_string()), ); @@ -236,7 +235,6 @@ impl Parser { .document .glossary .lock() - .unwrap() .assign_from_toml(value) .unwrap_or_else(|e| log::error!("{}", e)); diff --git a/src/references/bibliography.rs b/src/references/bibliography.rs index 923990b..c69cbc0 100644 --- a/src/references/bibliography.rs +++ b/src/references/bibliography.rs @@ -33,7 +33,6 @@ pub fn create_bib_list(entries: Vec) -> List { for entry in entries { entry .lock() - .unwrap() .raw_fields .insert("ord".to_string(), count.to_string()); list.add_item(get_item_for_entry(entry)); @@ -45,7 +44,7 @@ pub fn create_bib_list(entries: Vec) -> List { /// Returns the list item for a bib entry fn get_item_for_entry(entry: BibliographyEntryReference) -> ListItem { - let entry = entry.lock().unwrap(); + let entry = entry.lock(); match &entry.bib_type { BibliographyType::Article(a) => get_item_for_article(&*entry, a), diff --git a/src/references/configuration/keys.rs b/src/references/configuration/keys.rs index 7646ac5..cce5760 100644 --- a/src/references/configuration/keys.rs +++ b/src/references/configuration/keys.rs @@ -24,3 +24,8 @@ pub const PDF_MARGIN_RIGHT: &str = "pdf-margin-right"; pub const PDF_PAGE_HEIGHT: &str = "pdf-page-height"; pub const PDF_PAGE_WIDTH: &str = "pdf-page-width"; pub const PDF_PAGE_SCALE: &str = "pdf-page-scale"; + +// Image Options +pub const IMAGE_FORMAT: &str = "image-format"; +pub const IMAGE_MAX_WIDTH: &str = "image-max-width"; +pub const IMAGE_MAX_HEIGHT: &str = "image-max-height"; diff --git a/src/references/glossary.rs b/src/references/glossary.rs index 0751cbc..ba0238e 100644 --- a/src/references/glossary.rs +++ b/src/references/glossary.rs @@ -1,9 +1,10 @@ use crate::elements::{ Anchor, BoldText, Inline, ItalicText, Line, List, ListItem, PlainText, TextLine, }; +use parking_lot::Mutex; use std::cmp::Ordering; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use crate::bold_text; use crate::italic_text; @@ -110,11 +111,11 @@ impl GlossaryManager { /// Assignes entries to references pub fn assign_entries_to_references(&self) { for reference in &self.references { - let mut reference = reference.lock().unwrap(); + let mut reference = reference.lock(); if let Some(entry) = self.entries.get(&reference.short) { reference.entry = Some(Arc::clone(entry)); - let mut entry = entry.lock().unwrap(); + let mut entry = entry.lock(); if !entry.is_assigned { entry.is_assigned = true; @@ -130,13 +131,13 @@ impl GlossaryManager { let mut entries = self .entries .values() - .filter(|e| e.lock().unwrap().is_assigned) + .filter(|e| e.lock().is_assigned) .cloned() .collect::>>>(); entries.sort_by(|a, b| { - let a = a.lock().unwrap(); - let b = b.lock().unwrap(); + let a = a.lock(); + let b = b.lock(); if a.short > b.short { Ordering::Greater } else if a.short < b.short { @@ -146,7 +147,7 @@ impl GlossaryManager { } }); for entry in &entries { - let entry = entry.lock().unwrap(); + let entry = entry.lock(); let mut line = TextLine::new(); line.subtext.push(bold_text!(entry.short.clone())); line.subtext.push(plain_text!(" - ".to_string())); diff --git a/src/references/placeholders.rs b/src/references/placeholders.rs index 4aa19e7..f9c0be6 100644 --- a/src/references/placeholders.rs +++ b/src/references/placeholders.rs @@ -54,7 +54,7 @@ impl ProcessPlaceholders for Document { self.bibliography.get_entry_list_by_occurrence() )))), P_GLS => pholder.set_value(block!(Block::List( - self.glossary.lock().unwrap().create_glossary_list() + self.glossary.lock().create_glossary_list() ))), P_DATE => pholder.set_value(inline!(Inline::Plain(PlainText { value: get_date_string() diff --git a/src/utils/caching.rs b/src/utils/caching.rs index ce8fb3e..d9c6549 100644 --- a/src/utils/caching.rs +++ b/src/utils/caching.rs @@ -1,7 +1,6 @@ use platform_dirs::{AppDirs, AppUI}; -use std::collections::hash_map::DefaultHasher; +use sha2::Digest; use std::fs; -use std::hash::{Hash, Hasher}; use std::io; use std::path::PathBuf; @@ -23,9 +22,9 @@ impl CacheStorage { /// Returns the cache path for a given file pub fn get_file_path(&self, path: &PathBuf) -> PathBuf { - let mut hasher = DefaultHasher::new(); - path.hash(&mut hasher); - let mut file_name = PathBuf::from(format!("{:x}", hasher.finish())); + let mut hasher = sha2::Sha256::default(); + hasher.update(path.to_string_lossy().as_bytes()); + let mut file_name = PathBuf::from(format!("{:x}", hasher.finalize())); if let Some(extension) = path.extension() { file_name.set_extension(extension); diff --git a/src/utils/downloads.rs b/src/utils/downloads.rs index 4b0f535..c8d86f3 100644 --- a/src/utils/downloads.rs +++ b/src/utils/downloads.rs @@ -1,9 +1,10 @@ use crate::utils::caching::CacheStorage; use indicatif::{ProgressBar, ProgressStyle}; +use parking_lot::Mutex; use rayon::prelude::*; use std::fs::read; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; /// A manager for downloading urls in parallel #[derive(Clone, Debug)] @@ -35,7 +36,7 @@ impl DownloadManager { /// Downloads all download entries pub fn download_all(&self) { let pb = Arc::new(Mutex::new(ProgressBar::new(self.downloads.len() as u64))); - pb.lock().unwrap().set_style( + pb.lock().set_style( ProgressStyle::default_bar() .template("Fetching Embeds: [{bar:40.cyan/blue}]") .progress_chars("=> "), @@ -43,10 +44,10 @@ impl DownloadManager { let pb_cloned = Arc::clone(&pb); self.downloads.par_iter().for_each_with(pb_cloned, |pb, d| { - d.lock().unwrap().download(); - pb.lock().unwrap().inc(1); + d.lock().download(); + pb.lock().inc(1); }); - pb.lock().unwrap().finish_and_clear(); + pb.lock().finish_and_clear(); } } @@ -116,10 +117,14 @@ impl PendingDownload { /// Downloads the content from the given url fn download_content(&self) -> Option> { - reqwest::blocking::get(&self.path) - .ok() - .map(|c| c.bytes()) - .and_then(|b| b.ok()) - .map(|b| b.to_vec()) + download_path(self.path.clone()) } } + +pub fn download_path(path: String) -> Option> { + reqwest::blocking::get(&path) + .ok() + .map(|c| c.bytes()) + .and_then(|b| b.ok()) + .map(|b| b.to_vec()) +} diff --git a/src/utils/image_converting.rs b/src/utils/image_converting.rs new file mode 100644 index 0000000..d0d5aab --- /dev/null +++ b/src/utils/image_converting.rs @@ -0,0 +1,176 @@ +use crate::utils::caching::CacheStorage; +use crate::utils::downloads::download_path; +use image::imageops::FilterType; +use image::io::Reader as ImageReader; +use image::{GenericImageView, ImageFormat, ImageResult}; +use mime::Mime; +use parking_lot::Mutex; +use rayon::prelude::*; +use std::io; +use std::io::Cursor; +use std::path::PathBuf; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct ImageConverter { + images: Vec>>, + target_format: Option, + target_size: Option<(u32, u32)>, +} + +impl ImageConverter { + pub fn new() -> Self { + Self { + images: Vec::new(), + target_format: None, + target_size: None, + } + } + + pub fn set_target_size(&mut self, target_size: (u32, u32)) { + self.target_size = Some(target_size) + } + + pub fn set_target_format(&mut self, target_format: ImageFormat) { + self.target_format = Some(target_format); + } + + /// Adds an image to convert + pub fn add_image(&mut self, path: PathBuf) -> Arc> { + let image = Arc::new(Mutex::new(PendingImage::new(path))); + self.images.push(image.clone()); + + image + } + + /// Converts all images + pub fn convert_all(&mut self) { + self.images.par_iter().for_each(|image| { + let mut image = image.lock(); + if let Err(e) = image.convert(self.target_format.clone(), self.target_size.clone()) { + log::error!("Failed to embed image {:?}: {}", image.path, e) + } + }); + } +} + +#[derive(Clone, Debug)] +pub struct PendingImage { + pub path: PathBuf, + pub data: Option>, + cache: CacheStorage, + pub mime: Mime, +} + +impl PendingImage { + pub fn new(path: PathBuf) -> Self { + let mime = get_mime(&path); + + Self { + path, + data: None, + cache: CacheStorage::new(), + mime, + } + } + + /// Converts the image to the specified target format (specified by target_extension) + pub fn convert( + &mut self, + target_format: Option, + target_size: Option<(u32, u32)>, + ) -> ImageResult<()> { + let format = target_format + .or_else(|| { + self.path + .extension() + .and_then(|extension| ImageFormat::from_extension(extension)) + }) + .unwrap_or(ImageFormat::Png); + let output_path = self.get_output_path(format, target_size); + self.mime = get_mime(&output_path); + + if self.cache.has_file(&output_path) { + self.data = Some(self.cache.read(&output_path)?) + } else { + self.convert_image(format, target_size)?; + + if let Some(data) = &self.data { + self.cache.write(&output_path, data)?; + } + } + + Ok(()) + } + + /// Converts the image + fn convert_image( + &mut self, + format: ImageFormat, + target_size: Option<(u32, u32)>, + ) -> ImageResult<()> { + let mut image = ImageReader::open(self.get_path()?)?.decode()?; + + if let Some((width, height)) = target_size { + let dimensions = image.dimensions(); + + if dimensions.0 > width || dimensions.1 > height { + image = image.resize(width, height, FilterType::Lanczos3); + } + } + let data = Vec::new(); + let mut writer = Cursor::new(data); + + image.write_to(&mut writer, format)?; + self.data = Some(writer.into_inner()); + + Ok(()) + } + + /// Returns the path of the file + fn get_path(&self) -> io::Result { + if !self.path.exists() { + if self.cache.has_file(&self.path) { + return Ok(self.cache.get_file_path(&self.path)); + } + if let Some(data) = download_path(self.path.to_string_lossy().to_string()) { + self.cache.write(&self.path, data)?; + return Ok(self.cache.get_file_path(&self.path)); + } + } + Ok(self.path.clone()) + } + + /// Returns the output file name after converting the image + fn get_output_path( + &self, + target_format: ImageFormat, + target_size: Option<(u32, u32)>, + ) -> PathBuf { + let mut path = self.path.clone(); + let mut file_name = path.file_stem().unwrap().to_string_lossy().to_string(); + let extension = target_format.extensions_str()[0]; + let type_name = format!("{:?}", target_format); + + if let Some(target_size) = target_size { + file_name += &*format!("-{}-{}", target_size.0, target_size.1); + } + file_name += format!("-{}-converted", type_name).as_str(); + path.set_file_name(file_name); + path.set_extension(extension); + + path + } +} + +fn get_mime(path: &PathBuf) -> Mime { + let mime = mime_guess::from_ext( + path.clone() + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("png"), + ) + .first() + .unwrap_or(mime::IMAGE_PNG); + mime +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index d1229a1..7beaff7 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,4 +1,5 @@ pub mod caching; pub mod downloads; +pub mod image_converting; pub mod macros; pub mod parsing; From b9cf095cfa3a69e915d782fb287f4f794af84685 Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 16 Dec 2020 17:32:30 +0100 Subject: [PATCH 4/6] Add image filters Signed-off-by: trivernis --- README.md | 2 +- src/elements/mod.rs | 20 ++++++++++++++++++ src/parser/inline.rs | 26 +++++++++++++---------- src/utils/image_converting.rs | 39 ++++++++++++++++++++++++++++++++++- 4 files changed, 74 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index b8b8004..181b9a9 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ Hide a section (including subsections) in the TOC #[toc-hidden] Section Set the size of an image -!(url)[width = 42% height=auto] +!(url)[width = 42%, height=auto, grayscale, brightness=10, contrast=1.2] Set the source of a quote [author=Me date=[[date]] display="{{author}} - {{date}}"]> It's me diff --git a/src/elements/mod.rs b/src/elements/mod.rs index 12d4d8c..da3363a 100644 --- a/src/elements/mod.rs +++ b/src/elements/mod.rs @@ -710,6 +710,8 @@ impl Placeholder { pub trait Metadata { fn get_bool(&self, key: &str) -> bool; fn get_string(&self, key: &str) -> Option; + fn get_float(&self, key: &str) -> Option; + fn get_integer(&self, key: &str) -> Option; fn get_string_map(&self) -> HashMap; } @@ -730,6 +732,24 @@ impl Metadata for InlineMetadata { } } + fn get_float(&self, key: &str) -> Option { + if let Some(MetadataValue::Float(f)) = self.data.get(key) { + Some(*f) + } else if let Some(MetadataValue::Integer(i)) = self.data.get(key) { + Some(*i as f64) + } else { + None + } + } + + fn get_integer(&self, key: &str) -> Option { + if let Some(MetadataValue::Integer(i)) = self.data.get(key) { + Some(*i) + } else { + None + } + } + fn get_string_map(&self) -> HashMap { let mut string_map = HashMap::new(); for (k, v) in &self.data { diff --git a/src/parser/inline.rs b/src/parser/inline.rs index 9895c06..39bd2d6 100644 --- a/src/parser/inline.rs +++ b/src/parser/inline.rs @@ -140,21 +140,24 @@ impl ParseInline for Parser { self.ctm.seek_one()?; if let Ok(url) = self.parse_url(true) { - let metadata = if let Ok(meta) = self.parse_inline_metadata() { - Some(meta) - } else { - None - }; + let metadata = self.parse_inline_metadata().ok(); + let path = url.url.clone(); + let pending_image = self + .options + .document + .images + .lock() + .add_image(PathBuf::from(path)); + + if let Some(meta) = &metadata { + pending_image.lock().assign_from_meta(meta) + } + Ok(Image { url, metadata, - image_data: self - .options - .document - .images - .lock() - .add_image(PathBuf::from(path)), + image_data: pending_image, }) } else { Err(self.ctm.rewind_with_error(start_index)) @@ -502,6 +505,7 @@ impl ParseInline for Parser { self.ctm.seek_any(&INLINE_WHITESPACE)?; let mut value = MetadataValue::Bool(true); + if self.ctm.check_char(&EQ) { self.ctm.seek_one()?; self.ctm.seek_any(&INLINE_WHITESPACE)?; diff --git a/src/utils/image_converting.rs b/src/utils/image_converting.rs index d0d5aab..84e7cbf 100644 --- a/src/utils/image_converting.rs +++ b/src/utils/image_converting.rs @@ -1,3 +1,4 @@ +use crate::elements::Metadata; use crate::utils::caching::CacheStorage; use crate::utils::downloads::download_path; use image::imageops::FilterType; @@ -60,6 +61,9 @@ pub struct PendingImage { pub data: Option>, cache: CacheStorage, pub mime: Mime, + brightness: Option, + contrast: Option, + grayscale: bool, } impl PendingImage { @@ -71,9 +75,22 @@ impl PendingImage { data: None, cache: CacheStorage::new(), mime, + brightness: None, + contrast: None, + grayscale: false, } } + pub fn assign_from_meta(&mut self, meta: &M) { + if let Some(brightness) = meta.get_integer("brightness") { + self.brightness = Some(brightness as i32); + } + if let Some(contrast) = meta.get_float("contrast") { + self.contrast = Some(contrast as f32); + } + self.grayscale = meta.get_bool("grayscale"); + } + /// Converts the image to the specified target format (specified by target_extension) pub fn convert( &mut self, @@ -87,6 +104,7 @@ impl PendingImage { .and_then(|extension| ImageFormat::from_extension(extension)) }) .unwrap_or(ImageFormat::Png); + let output_path = self.get_output_path(format, target_size); self.mime = get_mime(&output_path); @@ -118,6 +136,18 @@ impl PendingImage { image = image.resize(width, height, FilterType::Lanczos3); } } + + if let Some(brightness) = self.brightness { + image = image.brighten(brightness); + } + + if let Some(contrast) = self.contrast { + image = image.adjust_contrast(contrast); + } + if self.grayscale { + image = image.grayscale(); + } + let data = Vec::new(); let mut writer = Cursor::new(data); @@ -155,7 +185,14 @@ impl PendingImage { if let Some(target_size) = target_size { file_name += &*format!("-{}-{}", target_size.0, target_size.1); } - file_name += format!("-{}-converted", type_name).as_str(); + if let Some(b) = self.brightness { + file_name += &*format!("-{}", b); + } + if let Some(c) = self.contrast { + file_name += &*format!("-{}", c); + } + + file_name += format!("-{}", type_name).as_str(); path.set_file_name(file_name); path.set_extension(extension); From 9fb6664a631dff2e27f22d38ab247dd9378c31e1 Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 16 Dec 2020 17:46:17 +0100 Subject: [PATCH 5/6] Add inverting images and progressbar for image processing Signed-off-by: trivernis --- README.md | 2 +- src/utils/image_converting.rs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 181b9a9..c8495c0 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ Hide a section (including subsections) in the TOC #[toc-hidden] Section Set the size of an image -!(url)[width = 42%, height=auto, grayscale, brightness=10, contrast=1.2] +!(url)[width = 42%, height=auto, grayscale, brightness=10, contrast=1.2, invert] Set the source of a quote [author=Me date=[[date]] display="{{author}} - {{date}}"]> It's me diff --git a/src/utils/image_converting.rs b/src/utils/image_converting.rs index 84e7cbf..874056e 100644 --- a/src/utils/image_converting.rs +++ b/src/utils/image_converting.rs @@ -4,6 +4,7 @@ use crate::utils::downloads::download_path; use image::imageops::FilterType; use image::io::Reader as ImageReader; use image::{GenericImageView, ImageFormat, ImageResult}; +use indicatif::{ProgressBar, ProgressStyle}; use mime::Mime; use parking_lot::Mutex; use rayon::prelude::*; @@ -46,12 +47,20 @@ impl ImageConverter { /// Converts all images pub fn convert_all(&mut self) { + let pb = Arc::new(Mutex::new(ProgressBar::new(self.images.len() as u64))); + pb.lock().set_style( + ProgressStyle::default_bar() + .template("Processing images: [{bar:40.cyan/blue}]") + .progress_chars("=> "), + ); self.images.par_iter().for_each(|image| { let mut image = image.lock(); if let Err(e) = image.convert(self.target_format.clone(), self.target_size.clone()) { log::error!("Failed to embed image {:?}: {}", image.path, e) } + pb.lock().tick(); }); + pb.lock().finish(); } } @@ -64,6 +73,7 @@ pub struct PendingImage { brightness: Option, contrast: Option, grayscale: bool, + invert: bool, } impl PendingImage { @@ -78,6 +88,7 @@ impl PendingImage { brightness: None, contrast: None, grayscale: false, + invert: false, } } @@ -89,6 +100,7 @@ impl PendingImage { self.contrast = Some(contrast as f32); } self.grayscale = meta.get_bool("grayscale"); + self.invert = meta.get_bool("invert"); } /// Converts the image to the specified target format (specified by target_extension) @@ -147,6 +159,9 @@ impl PendingImage { if self.grayscale { image = image.grayscale(); } + if self.invert { + image.invert(); + } let data = Vec::new(); let mut writer = Cursor::new(data); @@ -191,6 +206,7 @@ impl PendingImage { if let Some(c) = self.contrast { file_name += &*format!("-{}", c); } + file_name += &*format!("{}-{}", self.invert, self.grayscale); file_name += format!("-{}", type_name).as_str(); path.set_file_name(file_name); From 2b07fc39b2c3381e9501260d94e3bb74b48ad83d Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 16 Dec 2020 17:54:38 +0100 Subject: [PATCH 6/6] Add huerotate option to image Signed-off-by: trivernis --- README.md | 2 +- src/utils/image_converting.rs | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c8495c0..ebb20ab 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ Hide a section (including subsections) in the TOC #[toc-hidden] Section Set the size of an image -!(url)[width = 42%, height=auto, grayscale, brightness=10, contrast=1.2, invert] +!(url)[width = 42%, height=auto, brightness=10, contrast=1.2, huerotate=180, invert, grayscale] Set the source of a quote [author=Me date=[[date]] display="{{author}} - {{date}}"]> It's me diff --git a/src/utils/image_converting.rs b/src/utils/image_converting.rs index 874056e..3f8a526 100644 --- a/src/utils/image_converting.rs +++ b/src/utils/image_converting.rs @@ -72,6 +72,7 @@ pub struct PendingImage { pub mime: Mime, brightness: Option, contrast: Option, + huerotate: Option, grayscale: bool, invert: bool, } @@ -89,6 +90,7 @@ impl PendingImage { contrast: None, grayscale: false, invert: false, + huerotate: None, } } @@ -99,6 +101,9 @@ impl PendingImage { if let Some(contrast) = meta.get_float("contrast") { self.contrast = Some(contrast as f32); } + if let Some(huerotate) = meta.get_float("huerotate") { + self.huerotate = Some(huerotate as i32); + } self.grayscale = meta.get_bool("grayscale"); self.invert = meta.get_bool("invert"); } @@ -156,9 +161,15 @@ impl PendingImage { if let Some(contrast) = self.contrast { image = image.adjust_contrast(contrast); } + + if let Some(rotate) = self.huerotate { + image = image.huerotate(rotate); + } + if self.grayscale { image = image.grayscale(); } + if self.invert { image.invert(); } @@ -198,13 +209,16 @@ impl PendingImage { let type_name = format!("{:?}", target_format); if let Some(target_size) = target_size { - file_name += &*format!("-{}-{}", target_size.0, target_size.1); + file_name += &*format!("-w{}-h{}", target_size.0, target_size.1); } if let Some(b) = self.brightness { - file_name += &*format!("-{}", b); + file_name += &*format!("-b{}", b); } if let Some(c) = self.contrast { - file_name += &*format!("-{}", c); + file_name += &*format!("-c{}", c); + } + if let Some(h) = self.huerotate { + file_name += &*format!("-h{}", h); } file_name += &*format!("{}-{}", self.invert, self.grayscale);