Trivernis
/
snekdown
mirror of https://github.com/Trivernis/snekdown.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add progress bar to downloader

Browse Source 
Signed-off-by: trivernis <trivernis@protonmail.com>
feature/epub-rendering
trivernis 4 years ago
parent 234b11ee93
commit a226001b1c
Signed by: Trivernis
GPG Key ID: DFFFCC2C7A02DB45
6 changed files with 104 additions and 41 deletions
Show Stats Download Patch File Download Diff File

63
Cargo.lock generated
Unescape Escape View File

@ -150,6 +150,22 @@ dependencies = [
"winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "console"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"terminal_size 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
"termios 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-width 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "core-foundation"
version = "0.7.0"
@ -226,6 +242,11 @@ name = "either"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "encoding_rs"
version = "0.8.23"
@ -514,6 +535,17 @@ dependencies = [
"hashbrown 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "indicatif"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "inotify"
version = "0.7.1"
@ -768,6 +800,11 @@ dependencies = [
"libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "number_prefix"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "once_cell"
version = "1.4.0"
@ -1167,7 +1204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "snekdown"
version = "0.26.4"
version = "0.26.5"
dependencies = [
"asciimath-rs 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1179,6 +1216,7 @@ dependencies = [
"env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gh-emoji 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1286,6 +1324,23 @@ dependencies = [
"winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "terminal_size"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "termios"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "textwrap"
version = "0.11.0"
@ -1638,6 +1693,7 @@ dependencies = [
"checksum chrono-english 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4233ee19352739cfdcb5d7c2085005b166f6170ef2845ed9eef27a8fa5f95206"
"checksum clap 2.33.2 (registry+https://github.com/rust-lang/crates.io-index)" = "10040cdf04294b565d9e0319955430099ec3813a64c952b86a41200ad714ae48"
"checksum colored 1.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59"
"checksum console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2"
"checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171"
"checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
@ -1647,6 +1703,7 @@ dependencies = [
"checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
"checksum dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b"
"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
"checksum encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
"checksum encoding_rs 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171"
"checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
"checksum filetime 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e"
@ -1679,6 +1736,7 @@ dependencies = [
"checksum hyper-tls 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed"
"checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
"checksum indexmap 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7"
"checksum indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
"checksum inotify 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4816c66d2c8ae673df83366c18341538f234a26d65a9ecea5c348b453ac1d02f"
"checksum inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0"
"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
@ -1710,6 +1768,7 @@ dependencies = [
"checksum num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b"
"checksum num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
"checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
"checksum number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
"checksum once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"
"checksum onig 6.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd91ccd8a02fce2f7e8a86655aec67bc6c171e6f8e704118a0e8c4b866a05a8a"
"checksum onig_sys 69.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3814583fad89f3c60ae0701d80e87e1fd3028741723deda72d0d4a0ecf0cb0db"
@ -1764,6 +1823,8 @@ dependencies = [
"checksum syntect 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b57a45fdcf4891bc79f635be5c559210a4cfa464891f969724944c713282eedb"
"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
"checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
"checksum terminal_size 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13"
"checksum termios 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2"
"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
"checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"

5
Cargo.toml
Unescape Escape View File

@ -1,6 +1,6 @@
[package]
name = "snekdown"
version = "0.26.4"
version = "0.26.5"
authors = ["trivernis <trivernis@protonmail.com>"]
edition = "2018"
license-file = "LICENSE"
@ -41,4 +41,5 @@ base64 = "0.12.3"
rayon = "1.3.1"
maplit = "1.0.2"
log = "0.4.11"
env_logger = "0.7.1"
env_logger = "0.7.1"
indicatif = "0.15.0"

11
src/elements/mod.rs
Unescape Escape View File

@ -9,7 +9,6 @@ use asciimath_rs::elements::special::Expression;
use bibliographix::bib_manager::BibManager;
use bibliographix::bibliography::bibliography_entry::BibliographyEntryReference;
use bibliographix::references::bib_reference::BibRefAnchor;
use crossbeam_utils::sync::WaitGroup;
use std::collections::HashMap;
use std::iter::FromIterator;
use std::sync::atomic::{AtomicBool, Ordering};
@ -73,7 +72,7 @@ pub struct Document {
pub config: Configuration,
pub bibliography: BibManager,
pub downloads: Arc<Mutex<DownloadManager>>,
pub stylesheets: Vec<String>,
pub stylesheets: Vec<Arc<Mutex<PendingDownload>>>,
}
#[derive(Clone, Debug)]
@ -688,14 +687,8 @@ impl Into<HashMap<String, Value>> for InlineMetadata {
impl Image {
pub fn get_content(&self) -> Option<Vec<u8>> {
let mut download = self.download.lock().unwrap();
if let Some(wg) = &download.wg {
let wg = WaitGroup::clone(wg);
log::debug!("Waiting for content of {}", self.url.url.clone());
wg.wait();
}
let mut data = None;
std::mem::swap(&mut data, &mut download.data);
std::mem::swap(&mut data, &mut self.download.lock().unwrap().data);
data
}

15
src/format/html/to_html.rs
Unescape Escape View File

@ -8,7 +8,6 @@ use htmlescape::encode_attribute;
use minify::html::minify;
use std::io;
use std::sync::Arc;
use std::thread;
use syntect::highlighting::ThemeSet;
use syntect::html::highlighted_html_for_string;
use syntect::parsing::SyntaxSet;
@ -98,9 +97,7 @@ impl ToHtml for MetadataValue {
impl ToHtml for Document {
fn to_html(&self, writer: &mut HTMLWriter) -> io::Result<()> {
let downloads = Arc::clone(&self.downloads);
thread::spawn(move || {
downloads.lock().unwrap().download_all();
});
downloads.lock().unwrap().download_all();
let path = if let Some(path) = &self.path {
format!("path=\"{}\"", encode_attribute(path.as_str()))
} else {
@ -125,9 +122,13 @@ impl ToHtml for Document {
writer.write("</style>".to_string())?;
for stylesheet in &self.stylesheets {
writer.write("<style>".to_string())?;
writer.write(minify(stylesheet.as_str()))?;
writer.write("</style>".to_string())?;
let mut stylesheet = stylesheet.lock().unwrap();
let data = std::mem::replace(&mut stylesheet.data, None);
if let Some(data) = data {
writer.write("<style>".to_string())?;
writer.write(minify(String::from_utf8(data).unwrap().as_str()))?;
writer.write("</style>".to_string())?;
}
}
writer.write("</head><body><div class=\"content\">".to_string())?;
for element in &self.elements {

11
src/parser/mod.rs
Unescape Escape View File

@ -256,8 +256,13 @@ impl Parser {
}
fn import_stylesheet(&mut self, path: PathBuf) -> ParseResult<()> {
let content = self.import_text_file(path)?;
self.document.stylesheets.push(content);
self.document.stylesheets.push(
self.document
.downloads
.lock()
.unwrap()
.add_download(path.to_str().unwrap().to_string()),
);
Ok(())
}
@ -363,7 +368,7 @@ impl Parser {
self.wg = WaitGroup::new();
if !self.is_child {
for (path, file_type) in DEFAULT_IMPORTS {
if PathBuf::from(path).exists() {
if self.transform_path(path.to_string()).exists() {
self.import(
path.to_string(),
&maplit::hashmap! {"type".to_string() => Value::String(file_type.to_string())},

40
src/utils/downloads.rs
Unescape Escape View File

@ -1,4 +1,4 @@
use crossbeam_utils::sync::WaitGroup;
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use rayon::prelude::*;
use std::fs::read;
use std::path::PathBuf;
@ -20,17 +20,30 @@ impl DownloadManager {
/// Adds a new pending download
pub fn add_download(&mut self, path: String) -> Arc<Mutex<PendingDownload>> {
let pending = Arc::new(Mutex::new(PendingDownload::new(path)));
let pending = Arc::new(Mutex::new(PendingDownload::new(path.clone())));
self.downloads.push(Arc::clone(&pending));
log::debug!("Added download {}", path);
pending
}
/// Downloads all download entries
pub fn download_all(&self) {
self.downloads
.par_iter()
.for_each(|d| d.lock().unwrap().download())
let pb = Arc::new(Mutex::new(ProgressBar::new(self.downloads.len() as u64)));
pb.lock().unwrap().set_style(
ProgressStyle::default_bar()
.template("Reading Imports: [{bar:40.cyan/blue}]")
.progress_chars("=> "),
);
let pb_cloned = Arc::clone(&pb);
self.downloads.par_iter().for_each_with(pb_cloned, |pb, d| {
d.lock().unwrap().download();
pb.lock().unwrap().inc(1);
});
pb.lock()
.unwrap()
.finish_with_message("All downloads finished!");
}
}
@ -38,29 +51,18 @@ impl DownloadManager {
/// Download does not necessarily mean that it's not a local file
#[derive(Clone, Debug)]
pub struct PendingDownload {
path: String,
pub(crate) path: String,
pub(crate) data: Option<Vec<u8>>,
pub(crate) wg: Option<WaitGroup>,
}
impl PendingDownload {
pub fn new(path: String) -> Self {
Self {
path,
data: None,
wg: Some(WaitGroup::new()),
}
Self { path, data: None }
}
/// Downloads the file and writes the content to the content field
pub fn download(&mut self) {
let wg = std::mem::replace(&mut self.wg, None);
if let Some(wg) = wg {
log::debug!("Reading {}...", self.path);
self.data = self.read_content();
log::debug!("{} read!", self.path);
drop(wg);
}
self.data = self.read_content();
}
/// Reads the fiels content or downloads it if it doesn't exist in the filesystem

Write Preview
Loading…
Cancel
Save