From a226001b1ce3079e99cd2c5fe31cf02e3e0946c7 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 6 Sep 2020 13:45:45 +0200 Subject: [PATCH] Add progress bar to downloader Signed-off-by: trivernis --- Cargo.lock | 63 +++++++++++++++++++++++++++++++++++++- Cargo.toml | 5 +-- src/elements/mod.rs | 11 ++----- src/format/html/to_html.rs | 15 ++++----- src/parser/mod.rs | 11 +++++-- src/utils/downloads.rs | 40 ++++++++++++------------ 6 files changed, 104 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d17c77d..09e4748 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -150,6 +150,22 @@ dependencies = [ "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "console" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "terminal_size 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "termios 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "core-foundation" version = "0.7.0" @@ -226,6 +242,11 @@ name = "either" version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "encoding_rs" version = "0.8.23" @@ -514,6 +535,17 @@ dependencies = [ "hashbrown 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "indicatif" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "inotify" version = "0.7.1" @@ -768,6 +800,11 @@ dependencies = [ "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "once_cell" version = "1.4.0" @@ -1167,7 +1204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "snekdown" -version = "0.26.4" +version = "0.26.5" dependencies = [ "asciimath-rs 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1179,6 +1216,7 @@ dependencies = [ "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "gh-emoji 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1286,6 +1324,23 @@ dependencies = [ "winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "terminal_size" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termios" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -1638,6 +1693,7 @@ dependencies = [ "checksum chrono-english 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4233ee19352739cfdcb5d7c2085005b166f6170ef2845ed9eef27a8fa5f95206" "checksum clap 2.33.2 (registry+https://github.com/rust-lang/crates.io-index)" = "10040cdf04294b565d9e0319955430099ec3813a64c952b86a41200ad714ae48" "checksum colored 1.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59" +"checksum console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" "checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" "checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" @@ -1647,6 +1703,7 @@ dependencies = [ "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" +"checksum encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" "checksum encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" "checksum filetime 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e" @@ -1679,6 +1736,7 @@ dependencies = [ "checksum hyper-tls 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" "checksum indexmap 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7" +"checksum indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" "checksum inotify 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4816c66d2c8ae673df83366c18341538f234a26d65a9ecea5c348b453ac1d02f" "checksum inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0" "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" @@ -1710,6 +1768,7 @@ dependencies = [ "checksum num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" "checksum num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" "checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +"checksum number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" "checksum once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" "checksum onig 6.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd91ccd8a02fce2f7e8a86655aec67bc6c171e6f8e704118a0e8c4b866a05a8a" "checksum onig_sys 69.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3814583fad89f3c60ae0701d80e87e1fd3028741723deda72d0d4a0ecf0cb0db" @@ -1764,6 +1823,8 @@ dependencies = [ "checksum syntect 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b57a45fdcf4891bc79f635be5c559210a4cfa464891f969724944c713282eedb" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +"checksum terminal_size 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +"checksum termios 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" diff --git a/Cargo.toml b/Cargo.toml index 4e08b42..3dd9160 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "snekdown" -version = "0.26.4" +version = "0.26.5" authors = ["trivernis "] edition = "2018" license-file = "LICENSE" @@ -41,4 +41,5 @@ base64 = "0.12.3" rayon = "1.3.1" maplit = "1.0.2" log = "0.4.11" -env_logger = "0.7.1" \ No newline at end of file +env_logger = "0.7.1" +indicatif = "0.15.0" \ No newline at end of file diff --git a/src/elements/mod.rs b/src/elements/mod.rs index 6a9e5c1..fc3f42a 100644 --- a/src/elements/mod.rs +++ b/src/elements/mod.rs @@ -9,7 +9,6 @@ use asciimath_rs::elements::special::Expression; use bibliographix::bib_manager::BibManager; use bibliographix::bibliography::bibliography_entry::BibliographyEntryReference; use bibliographix::references::bib_reference::BibRefAnchor; -use crossbeam_utils::sync::WaitGroup; use std::collections::HashMap; use std::iter::FromIterator; use std::sync::atomic::{AtomicBool, Ordering}; @@ -73,7 +72,7 @@ pub struct Document { pub config: Configuration, pub bibliography: BibManager, pub downloads: Arc>, - pub stylesheets: Vec, + pub stylesheets: Vec>>, } #[derive(Clone, Debug)] @@ -688,14 +687,8 @@ impl Into> for InlineMetadata { impl Image { pub fn get_content(&self) -> Option> { - let mut download = self.download.lock().unwrap(); - if let Some(wg) = &download.wg { - let wg = WaitGroup::clone(wg); - log::debug!("Waiting for content of {}", self.url.url.clone()); - wg.wait(); - } let mut data = None; - std::mem::swap(&mut data, &mut download.data); + std::mem::swap(&mut data, &mut self.download.lock().unwrap().data); data } diff --git a/src/format/html/to_html.rs b/src/format/html/to_html.rs index 871623e..eebc6f6 100644 --- a/src/format/html/to_html.rs +++ b/src/format/html/to_html.rs @@ -8,7 +8,6 @@ use htmlescape::encode_attribute; use minify::html::minify; use std::io; use std::sync::Arc; -use std::thread; use syntect::highlighting::ThemeSet; use syntect::html::highlighted_html_for_string; use syntect::parsing::SyntaxSet; @@ -98,9 +97,7 @@ impl ToHtml for MetadataValue { impl ToHtml for Document { fn to_html(&self, writer: &mut HTMLWriter) -> io::Result<()> { let downloads = Arc::clone(&self.downloads); - thread::spawn(move || { - downloads.lock().unwrap().download_all(); - }); + downloads.lock().unwrap().download_all(); let path = if let Some(path) = &self.path { format!("path=\"{}\"", encode_attribute(path.as_str())) } else { @@ -125,9 +122,13 @@ impl ToHtml for Document { writer.write("".to_string())?; for stylesheet in &self.stylesheets { - writer.write("".to_string())?; + let mut stylesheet = stylesheet.lock().unwrap(); + let data = std::mem::replace(&mut stylesheet.data, None); + if let Some(data) = data { + writer.write("".to_string())?; + } } writer.write("
".to_string())?; for element in &self.elements { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 0363794..bbf74ff 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -256,8 +256,13 @@ impl Parser { } fn import_stylesheet(&mut self, path: PathBuf) -> ParseResult<()> { - let content = self.import_text_file(path)?; - self.document.stylesheets.push(content); + self.document.stylesheets.push( + self.document + .downloads + .lock() + .unwrap() + .add_download(path.to_str().unwrap().to_string()), + ); Ok(()) } @@ -363,7 +368,7 @@ impl Parser { self.wg = WaitGroup::new(); if !self.is_child { for (path, file_type) in DEFAULT_IMPORTS { - if PathBuf::from(path).exists() { + if self.transform_path(path.to_string()).exists() { self.import( path.to_string(), &maplit::hashmap! {"type".to_string() => Value::String(file_type.to_string())}, diff --git a/src/utils/downloads.rs b/src/utils/downloads.rs index 2defcc4..2a4da25 100644 --- a/src/utils/downloads.rs +++ b/src/utils/downloads.rs @@ -1,4 +1,4 @@ -use crossbeam_utils::sync::WaitGroup; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use rayon::prelude::*; use std::fs::read; use std::path::PathBuf; @@ -20,17 +20,30 @@ impl DownloadManager { /// Adds a new pending download pub fn add_download(&mut self, path: String) -> Arc> { - let pending = Arc::new(Mutex::new(PendingDownload::new(path))); + let pending = Arc::new(Mutex::new(PendingDownload::new(path.clone()))); self.downloads.push(Arc::clone(&pending)); + log::debug!("Added download {}", path); pending } /// Downloads all download entries pub fn download_all(&self) { - self.downloads - .par_iter() - .for_each(|d| d.lock().unwrap().download()) + let pb = Arc::new(Mutex::new(ProgressBar::new(self.downloads.len() as u64))); + pb.lock().unwrap().set_style( + ProgressStyle::default_bar() + .template("Reading Imports: [{bar:40.cyan/blue}]") + .progress_chars("=> "), + ); + let pb_cloned = Arc::clone(&pb); + + self.downloads.par_iter().for_each_with(pb_cloned, |pb, d| { + d.lock().unwrap().download(); + pb.lock().unwrap().inc(1); + }); + pb.lock() + .unwrap() + .finish_with_message("All downloads finished!"); } } @@ -38,29 +51,18 @@ impl DownloadManager { /// Download does not necessarily mean that it's not a local file #[derive(Clone, Debug)] pub struct PendingDownload { - path: String, + pub(crate) path: String, pub(crate) data: Option>, - pub(crate) wg: Option, } impl PendingDownload { pub fn new(path: String) -> Self { - Self { - path, - data: None, - wg: Some(WaitGroup::new()), - } + Self { path, data: None } } /// Downloads the file and writes the content to the content field pub fn download(&mut self) { - let wg = std::mem::replace(&mut self.wg, None); - if let Some(wg) = wg { - log::debug!("Reading {}...", self.path); - self.data = self.read_content(); - log::debug!("{} read!", self.path); - drop(wg); - } + self.data = self.read_content(); } /// Reads the fiels content or downloads it if it doesn't exist in the filesystem