Add progress bar to downloader

Signed-off-by: trivernis <trivernis@protonmail.com>

Cargo.lock

@@ -150,6 +150,22 @@ dependencies = [
  "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "console"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "terminal_size 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "termios 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-width 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.7.0"
@@ -226,6 +242,11 @@ name = "either"
 version = "1.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.23"
@@ -514,6 +535,17 @@ dependencies = [
  "hashbrown 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "inotify"
 version = "0.7.1"
@@ -768,6 +800,11 @@ dependencies = [
  "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "once_cell"
 version = "1.4.0"
@@ -1167,7 +1204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "snekdown"
-version = "0.26.4"
+version = "0.26.5"
 dependencies = [
  "asciimath-rs 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1179,6 +1216,7 @@ dependencies = [
  "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gh-emoji 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1286,6 +1324,23 @@ dependencies = [
  "winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "terminal_size"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "termios"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"
@@ -1638,6 +1693,7 @@ dependencies = [
 "checksum chrono-english 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4233ee19352739cfdcb5d7c2085005b166f6170ef2845ed9eef27a8fa5f95206"
 "checksum clap 2.33.2 (registry+https://github.com/rust-lang/crates.io-index)" = "10040cdf04294b565d9e0319955430099ec3813a64c952b86a41200ad714ae48"
 "checksum colored 1.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59"
+"checksum console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2"
 "checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171"
 "checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
 "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
@@ -1647,6 +1703,7 @@ dependencies = [
 "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
 "checksum dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b"
 "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
+"checksum encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
 "checksum encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171"
 "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
 "checksum filetime 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e"
@@ -1679,6 +1736,7 @@ dependencies = [
 "checksum hyper-tls 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed"
 "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
 "checksum indexmap 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7"
+"checksum indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
 "checksum inotify 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4816c66d2c8ae673df83366c18341538f234a26d65a9ecea5c348b453ac1d02f"
 "checksum inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0"
 "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
@@ -1710,6 +1768,7 @@ dependencies = [
 "checksum num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b"
 "checksum num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
 "checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+"checksum number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
 "checksum once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"
 "checksum onig 6.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd91ccd8a02fce2f7e8a86655aec67bc6c171e6f8e704118a0e8c4b866a05a8a"
 "checksum onig_sys 69.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3814583fad89f3c60ae0701d80e87e1fd3028741723deda72d0d4a0ecf0cb0db"
@@ -1764,6 +1823,8 @@ dependencies = [
 "checksum syntect 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b57a45fdcf4891bc79f635be5c559210a4cfa464891f969724944c713282eedb"
 "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
+"checksum terminal_size 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13"
+"checksum termios 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2"
 "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
 "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
 "checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snekdown"
-version = "0.26.4"
+version = "0.26.5"
 authors = ["trivernis <trivernis@protonmail.com>"]
 edition = "2018"
 license-file = "LICENSE"
@@ -41,4 +41,5 @@ base64 = "0.12.3"
 rayon = "1.3.1"
 maplit = "1.0.2"
 log = "0.4.11"
-env_logger = "0.7.1"
+env_logger = "0.7.1"
+indicatif = "0.15.0"

src/elements/mod.rs

@@ -9,7 +9,6 @@ use asciimath_rs::elements::special::Expression;
 use bibliographix::bib_manager::BibManager;
 use bibliographix::bibliography::bibliography_entry::BibliographyEntryReference;
 use bibliographix::references::bib_reference::BibRefAnchor;
-use crossbeam_utils::sync::WaitGroup;
 use std::collections::HashMap;
 use std::iter::FromIterator;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -73,7 +72,7 @@ pub struct Document {
     pub config: Configuration,
     pub bibliography: BibManager,
     pub downloads: Arc<Mutex<DownloadManager>>,
-    pub stylesheets: Vec<String>,
+    pub stylesheets: Vec<Arc<Mutex<PendingDownload>>>,
 }
 
 #[derive(Clone, Debug)]
@@ -688,14 +687,8 @@ impl Into<HashMap<String, Value>> for InlineMetadata {
 
 impl Image {
     pub fn get_content(&self) -> Option<Vec<u8>> {
-        let mut download = self.download.lock().unwrap();
-        if let Some(wg) = &download.wg {
-            let wg = WaitGroup::clone(wg);
-            log::debug!("Waiting for content of {}", self.url.url.clone());
-            wg.wait();
-        }
         let mut data = None;
-        std::mem::swap(&mut data, &mut download.data);
+        std::mem::swap(&mut data, &mut self.download.lock().unwrap().data);
 
         data
     }

src/format/html/to_html.rs

@@ -8,7 +8,6 @@ use htmlescape::encode_attribute;
 use minify::html::minify;
 use std::io;
 use std::sync::Arc;
-use std::thread;
 use syntect::highlighting::ThemeSet;
 use syntect::html::highlighted_html_for_string;
 use syntect::parsing::SyntaxSet;
@@ -98,9 +97,7 @@ impl ToHtml for MetadataValue {
 impl ToHtml for Document {
     fn to_html(&self, writer: &mut HTMLWriter) -> io::Result<()> {
         let downloads = Arc::clone(&self.downloads);
-        thread::spawn(move || {
+        downloads.lock().unwrap().download_all();
-            downloads.lock().unwrap().download_all();
-        });
         let path = if let Some(path) = &self.path {
             format!("path=\"{}\"", encode_attribute(path.as_str()))
         } else {
@@ -125,9 +122,13 @@ impl ToHtml for Document {
             writer.write("</style>".to_string())?;
 
             for stylesheet in &self.stylesheets {
-                writer.write("<style>".to_string())?;
+                let mut stylesheet = stylesheet.lock().unwrap();
-                writer.write(minify(stylesheet.as_str()))?;
+                let data = std::mem::replace(&mut stylesheet.data, None);
-                writer.write("</style>".to_string())?;
+                if let Some(data) = data {
+                    writer.write("<style>".to_string())?;
+                    writer.write(minify(String::from_utf8(data).unwrap().as_str()))?;
+                    writer.write("</style>".to_string())?;
+                }
             }
             writer.write("</head><body><div class=\"content\">".to_string())?;
             for element in &self.elements {

src/parser/mod.rs

@@ -256,8 +256,13 @@ impl Parser {
     }
 
     fn import_stylesheet(&mut self, path: PathBuf) -> ParseResult<()> {
-        let content = self.import_text_file(path)?;
+        self.document.stylesheets.push(
-        self.document.stylesheets.push(content);
+            self.document
+                .downloads
+                .lock()
+                .unwrap()
+                .add_download(path.to_str().unwrap().to_string()),
+        );
 
         Ok(())
     }
@@ -363,7 +368,7 @@ impl Parser {
         self.wg = WaitGroup::new();
         if !self.is_child {
             for (path, file_type) in DEFAULT_IMPORTS {
-                if PathBuf::from(path).exists() {
+                if self.transform_path(path.to_string()).exists() {
                     self.import(
                         path.to_string(),
                         &maplit::hashmap! {"type".to_string() => Value::String(file_type.to_string())},

src/utils/downloads.rs

@@ -1,4 +1,4 @@
-use crossbeam_utils::sync::WaitGroup;
+use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use rayon::prelude::*;
 use std::fs::read;
 use std::path::PathBuf;
@@ -20,17 +20,30 @@ impl DownloadManager {
 
     /// Adds a new pending download
     pub fn add_download(&mut self, path: String) -> Arc<Mutex<PendingDownload>> {
-        let pending = Arc::new(Mutex::new(PendingDownload::new(path)));
+        let pending = Arc::new(Mutex::new(PendingDownload::new(path.clone())));
         self.downloads.push(Arc::clone(&pending));
+        log::debug!("Added download {}", path);
 
         pending
     }
 
     /// Downloads all download entries
     pub fn download_all(&self) {
-        self.downloads
+        let pb = Arc::new(Mutex::new(ProgressBar::new(self.downloads.len() as u64)));
-            .par_iter()
+        pb.lock().unwrap().set_style(
-            .for_each(|d| d.lock().unwrap().download())
+            ProgressStyle::default_bar()
+                .template("Reading Imports: [{bar:40.cyan/blue}]")
+                .progress_chars("=> "),
+        );
+        let pb_cloned = Arc::clone(&pb);
+
+        self.downloads.par_iter().for_each_with(pb_cloned, |pb, d| {
+            d.lock().unwrap().download();
+            pb.lock().unwrap().inc(1);
+        });
+        pb.lock()
+            .unwrap()
+            .finish_with_message("All downloads finished!");
     }
 }
 
@@ -38,29 +51,18 @@ impl DownloadManager {
 /// Download does not necessarily mean that it's not a local file
 #[derive(Clone, Debug)]
 pub struct PendingDownload {
-    path: String,
+    pub(crate) path: String,
     pub(crate) data: Option<Vec<u8>>,
-    pub(crate) wg: Option<WaitGroup>,
 }
 
 impl PendingDownload {
     pub fn new(path: String) -> Self {
-        Self {
+        Self { path, data: None }
-            path,
-            data: None,
-            wg: Some(WaitGroup::new()),
-        }
     }
 
     /// Downloads the file and writes the content to the content field
     pub fn download(&mut self) {
-        let wg = std::mem::replace(&mut self.wg, None);
+        self.data = self.read_content();
-        if let Some(wg) = wg {
-            log::debug!("Reading {}...", self.path);
-            self.data = self.read_content();
-            log::debug!("{} read!", self.path);
-            drop(wg);
-        }
     }
 
     /// Reads the fiels content or downloads it if it doesn't exist in the filesystem