From dc884890d4e085605ebe8cf2555be060262e3904 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 6 Sep 2020 16:41:11 +0200 Subject: [PATCH] Add download cache and refactor parser creation Signed-off-by: trivernis --- Cargo.lock | 85 ++++++++++++++++- Cargo.toml | 5 +- README.md | 7 +- src/elements/mod.rs | 20 ++-- src/main.rs | 11 ++- src/parser/inline.rs | 17 +++- src/parser/line.rs | 4 +- src/parser/mod.rs | 206 ++++++++++++++++------------------------- src/utils/downloads.rs | 65 ++++++++++++- tests/parsing_tests.rs | 5 - 10 files changed, 268 insertions(+), 157 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f49985..cf805bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,16 @@ dependencies = [ "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "asciimath-rs" version = "0.5.7" @@ -76,6 +86,16 @@ name = "bitflags" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "blake2b_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayref 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "constant_time_eq 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bumpalo" version = "3.4.0" @@ -166,6 +186,11 @@ dependencies = [ "winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "core-foundation" version = "0.7.0" @@ -232,6 +257,25 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "dirs-sys 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "dirs-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_users 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "dtoa" version = "0.4.6" @@ -914,6 +958,14 @@ name = "pkg-config" version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "platform-dirs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "plist" version = "1.0.0" @@ -1040,6 +1092,16 @@ name = "redox_syscall" version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.57 (registry+https://github.com/rust-lang/crates.io-index)", + "rust-argon2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex" version = "1.3.9" @@ -1098,6 +1160,17 @@ dependencies = [ "winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rust-argon2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "blake2b_simd 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", + "constant_time_eq 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1204,7 +1277,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "snekdown" -version = "0.26.6" +version = "0.27.0" dependencies = [ "asciimath-rs 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1224,6 +1297,7 @@ dependencies = [ "mime_guess 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "minify 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "platform-dirs 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1676,6 +1750,8 @@ dependencies = [ "checksum adler 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" "checksum aho-corasick 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)" = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum arrayref 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +"checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" "checksum asciimath-rs 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)" = "033d0b4b21852b09c3275d631934891f9b8edb42d7d91349bc50d30f19cc2fd3" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" @@ -1683,6 +1759,7 @@ dependencies = [ "checksum bibliographix 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "39034545c510b822e3e5bd76147f869bae617d52961add6313ad0696084585c1" "checksum bincode 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum blake2b_simd 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" "checksum bumpalo 3.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" "checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" "checksum bytes 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" @@ -1694,6 +1771,7 @@ dependencies = [ "checksum clap 2.33.2 (registry+https://github.com/rust-lang/crates.io-index)" = "10040cdf04294b565d9e0319955430099ec3813a64c952b86a41200ad714ae48" "checksum colored 1.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59" "checksum console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +"checksum constant_time_eq 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" "checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" "checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" @@ -1701,6 +1779,8 @@ dependencies = [ "checksum crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" "checksum crossbeam-queue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +"checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +"checksum dirs-sys 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" "checksum dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" "checksum encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" @@ -1783,6 +1863,7 @@ dependencies = [ "checksum pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" "checksum pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" "checksum pkg-config 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)" = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" +"checksum platform-dirs 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f1e6f10c0c97e3d27b298374c2c67a057216c98e0a86c44df6bcd1f02bacbe38" "checksum plist 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b336d94e8e4ce29bf15bba393164629764744c567e8ad306cc1fdd0119967fd" "checksum ppv-lite86 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" "checksum proc-macro-error 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" @@ -1797,10 +1878,12 @@ dependencies = [ "checksum rayon 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "62f02856753d04e03e26929f820d0a0a337ebe71f849801eea335d464b349080" "checksum rayon-core 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e92e15d89083484e11353891f1af602cc661426deb9564c298b270c726973280" "checksum redox_syscall 0.1.57 (registry+https://github.com/rust-lang/crates.io-index)" = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +"checksum redox_users 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" "checksum regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" "checksum regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)" = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" "checksum remove_dir_all 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" "checksum reqwest 0.10.7 (registry+https://github.com/rust-lang/crates.io-index)" = "12427a5577082c24419c9c417db35cfeb65962efc7675bb6b0d5f1f9d315bfe6" +"checksum rust-argon2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" "checksum ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" "checksum safemem 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" "checksum same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" diff --git a/Cargo.toml b/Cargo.toml index 5fdab17..e04c5e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "snekdown" -version = "0.26.6" +version = "0.27.0" authors = ["trivernis "] edition = "2018" license-file = "LICENSE" @@ -42,4 +42,5 @@ rayon = "1.3.1" maplit = "1.0.2" log = "0.4.11" env_logger = "0.7.1" -indicatif = "0.15.0" \ No newline at end of file +indicatif = "0.15.0" +platform-dirs = "0.2.0" \ No newline at end of file diff --git a/README.md b/README.md index bc51965..8f0701b 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,12 @@ for my needs. ``` USAGE: - snekdown [OPTIONS] [SUBCOMMAND] + snekdown [FLAGS] [OPTIONS] [SUBCOMMAND] FLAGS: - -h, --help Prints help information - -V, --version Prints version information + -h, --help Prints help information + --no-cache Don't use the cache + -V, --version Prints version information OPTIONS: -f, --format the output format [default: html] diff --git a/src/elements/mod.rs b/src/elements/mod.rs index fc3f42a..8a22fc9 100644 --- a/src/elements/mod.rs +++ b/src/elements/mod.rs @@ -289,10 +289,11 @@ pub struct CharacterCode { // implementations impl Document { - pub fn new(is_root: bool) -> Self { + /// Creates a new parent document + pub fn new() -> Self { Self { elements: Vec::new(), - is_root, + is_root: true, path: None, placeholders: Vec::new(), config: Configuration::default(), @@ -302,20 +303,17 @@ impl Document { } } - pub fn new_with_manager( - is_root: bool, - bibliography: BibManager, - downloads: Arc>, - ) -> Self { + /// Creates a new child document + pub fn create_child(&self) -> Self { Self { elements: Vec::new(), - is_root, + is_root: false, path: None, placeholders: Vec::new(), - config: Configuration::default(), - bibliography, + config: self.config.clone(), + bibliography: self.bibliography.create_child(), stylesheets: Vec::new(), - downloads, + downloads: Arc::clone(&self.downloads), } } diff --git a/src/main.rs b/src/main.rs index 0eee2f4..0d71f04 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ use log::{Level, LevelFilter}; use notify::{watcher, RecursiveMode, Watcher}; use snekdown::format::html::html_writer::HTMLWriter; use snekdown::format::html::to_html::ToHtml; +use snekdown::parser::ParserOptions; use snekdown::Parser; use std::fs::OpenOptions; use std::io::BufWriter; @@ -26,6 +27,10 @@ struct Opt { #[structopt(short, long, default_value = "html")] format: String, + /// Don't use the cache + #[structopt(long)] + no_cache: bool, + #[structopt(subcommand)] sub_command: Option, } @@ -109,7 +114,11 @@ fn watch(opt: &Opt) { /// Renders the document to the output path fn render(opt: &Opt) -> Parser { let start = Instant::now(); - let mut parser = Parser::new_from_file(opt.input.clone()).unwrap(); + let mut parser = Parser::with_defaults( + ParserOptions::default() + .add_path(opt.input.clone()) + .use_cache(!opt.no_cache), + ); let document = parser.parse(); log::info!("Parsing took: {:?}", start.elapsed()); diff --git a/src/parser/inline.rs b/src/parser/inline.rs index 5393cd1..ccbed68 100644 --- a/src/parser/inline.rs +++ b/src/parser/inline.rs @@ -136,7 +136,13 @@ impl ParseInline for Parser { Ok(Image { url, metadata, - download: self.document.downloads.lock().unwrap().add_download(path), + download: self + .options + .document + .downloads + .lock() + .unwrap() + .add_download(path), }) } else { Err(self.ctm.rewind_with_error(start_index)) @@ -328,10 +334,11 @@ impl ParseInline for Parser { let bib_ref = BibRef::new(key.clone()); let ref_entry = Arc::new(RwLock::new(BibReference::new( key, - self.document.config.get_ref_entry(BIB_REF_DISPLAY), + self.options.document.config.get_ref_entry(BIB_REF_DISPLAY), bib_ref.anchor(), ))); - self.document + self.options + .document .bibliography .root_ref_anchor() .lock() @@ -501,7 +508,9 @@ impl ParseInline for Parser { }; let placeholder = Arc::new(RwLock::new(Placeholder::new(name, metadata))); - self.document.add_placeholder(Arc::clone(&placeholder)); + self.options + .document + .add_placeholder(Arc::clone(&placeholder)); Ok(placeholder) } diff --git a/src/parser/line.rs b/src/parser/line.rs index 5c52a56..aa9170b 100644 --- a/src/parser/line.rs +++ b/src/parser/line.rs @@ -237,7 +237,8 @@ impl ParseLine for Parser { } }; - self.document + self.options + .document .bibliography .entry_dictionary() .lock() @@ -246,6 +247,7 @@ impl ParseLine for Parser { Ok(BibEntry { entry: self + .options .document .bibliography .entry_dictionary() diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b21e36e..aad548b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8,16 +8,13 @@ use crate::elements::{Document, ImportAnchor}; use crate::references::configuration::keys::{ IMP_BIBLIOGRAPHY, IMP_CONFIGS, IMP_IGNORE, IMP_STYLESHEETS, }; -use crate::references::configuration::{Configuration, Value}; -use crate::utils::downloads::DownloadManager; -use bibliographix::bib_manager::BibManager; +use crate::references::configuration::Value; use charred::tapemachine::{CharTapeMachine, TapeError, TapeResult}; use crossbeam_utils::sync::WaitGroup; use regex::Regex; use std::collections::HashMap; use std::fs::{read_to_string, File}; -use std::io; -use std::io::{BufRead, BufReader, Cursor}; +use std::io::BufReader; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; use std::thread; @@ -33,133 +30,87 @@ const DEFAULT_IMPORTS: &'static [(&str, &str)] = &[ ("style.css", "stylesheet"), ]; +#[derive(Clone, Debug)] +pub struct ParserOptions { + pub path: Option, + pub paths: Arc>>, + pub document: Document, + pub is_child: bool, +} + +impl Default for ParserOptions { + fn default() -> Self { + Self { + path: None, + paths: Arc::new(Mutex::new(Vec::new())), + document: Document::new(), + is_child: false, + } + } +} + +impl ParserOptions { + /// Adds a path to the parser options + pub fn add_path(mut self, path: PathBuf) -> Self { + self.path = Some(path.clone()); + self.paths.lock().unwrap().push(path); + + self + } + + pub fn use_cache(self, value: bool) -> Self { + self.document.downloads.lock().unwrap().use_cache = value; + + self + } +} + pub struct Parser { + pub(crate) options: ParserOptions, pub(crate) ctm: CharTapeMachine, section_nesting: u8, sections: Vec, section_return: Option, - path: Option, - paths: Arc>>, wg: WaitGroup, - is_child: bool, pub(crate) block_break_at: Vec, pub(crate) inline_break_at: Vec, - pub(crate) document: Document, pub(crate) parse_variables: bool, } impl Parser { - /// Creates a new parser from a path - pub fn new_from_file(path: PathBuf) -> Result { - let f = File::open(&path)?; - Ok(Self::create( - Some(PathBuf::from(path)), - Arc::new(Mutex::new(Vec::new())), - false, - Box::new(BufReader::new(f)), - BibManager::new(), - Arc::new(Mutex::new(DownloadManager::new())), - )) - } + /// Creates a new parser with the default values given + pub fn with_defaults(options: ParserOptions) -> Self { + let text = if let Some(path) = &options.path { + let mut text = read_to_string(&path).unwrap(); + if text.chars().last() != Some('\n') { + text.push('\n'); + } - /// Creates a new parser with text being the markdown text - pub fn new(text: String, path: Option) -> Self { - let text_bytes = text.as_bytes(); - let path = if let Some(inner_path) = path { - Some(PathBuf::from(inner_path)) + text } else { - None + "".to_string() }; - Parser::create( - path, - Arc::new(Mutex::new(Vec::new())), - false, - Box::new(Cursor::new(text_bytes.to_vec())), - BibManager::new(), - Arc::new(Mutex::new(DownloadManager::new())), - ) - } - - /// Creates a child parser from string text - pub fn child( - text: String, - path: PathBuf, - paths: Arc>>, - bib_manager: BibManager, - download_manager: Arc>, - ) -> Self { - let text_bytes = text.as_bytes(); - Self::create( - Some(PathBuf::from(path)), - paths, - true, - Box::new(Cursor::new(text_bytes.to_vec())), - bib_manager, - download_manager, - ) - } - - /// Creates a child parser from a file - pub fn child_from_file( - path: PathBuf, - paths: Arc>>, - bib_manager: BibManager, - download_manager: Arc>, - ) -> Result { - let f = File::open(&path)?; - Ok(Self::create( - Some(PathBuf::from(path)), - paths, - true, - Box::new(BufReader::new(f)), - bib_manager, - download_manager, - )) - } - - fn create( - path: Option, - paths: Arc>>, - is_child: bool, - mut reader: Box, - bib_manager: BibManager, - download_manager: Arc>, - ) -> Self { - if let Some(path) = path.clone() { - paths.lock().unwrap().push(path.clone()) - } - let mut text = String::new(); - reader - .read_to_string(&mut text) - .expect("Failed to read file"); - if text.chars().last() != Some('\n') { - text.push('\n'); - } - - let document = Document::new_with_manager(!is_child, bib_manager, download_manager); Self { + options, sections: Vec::new(), section_nesting: 0, section_return: None, - path, - paths, wg: WaitGroup::new(), - is_child, ctm: CharTapeMachine::new(text.chars().collect()), inline_break_at: Vec::new(), block_break_at: Vec::new(), - document, parse_variables: false, } } - pub fn set_config(&mut self, config: Configuration) { - self.document.config = config; - } + /// Creates a new child parser + fn create_child(&self, path: PathBuf) -> Self { + let mut options = self.options.clone().add_path(path.clone()); + options.document = self.options.document.create_child(); + options.document.path = Some(path.to_str().unwrap().to_string()); + options.is_child = true; - /// Returns the import paths of the parser - pub fn get_paths(&self) -> Vec { - self.paths.lock().unwrap().clone() + Self::with_defaults(options) } /// Returns a string of the current position in the file @@ -174,7 +125,7 @@ impl Parser { while text_unil[inline_pos] != LB { inline_pos += 1; } - if let Some(path) = &self.path { + if let Some(path) = &self.options.path { format!("{}:{}:{}", path.to_str().unwrap(), line_number, inline_pos) } else { format!("{}:{}", line_number, inline_pos) @@ -186,7 +137,7 @@ impl Parser { let mut path = PathBuf::from(path); if !path.is_absolute() { - if let Some(selfpath) = &self.path { + if let Some(selfpath) = &self.options.path { if let Some(dir) = selfpath.parent() { path = PathBuf::new().join(dir).join(path); } @@ -207,7 +158,7 @@ impl Parser { return Err(self.ctm.assert_error(None)); } { - let mut paths = self.paths.lock().unwrap(); + let mut paths = self.options.paths.lock().unwrap(); if paths.iter().find(|item| **item == path) != None { log::warn!( "Import of \"{}\" failed: Already imported.\n\t--> {}\n", @@ -221,16 +172,10 @@ impl Parser { let anchor = Arc::new(RwLock::new(ImportAnchor::new())); let anchor_clone = Arc::clone(&anchor); let wg = self.wg.clone(); - let paths = Arc::clone(&self.paths); - let config = self.document.config.clone(); - let bibliography = self.document.bibliography.create_child(); - let download_manager = Arc::clone(&self.document.downloads); + let mut chid_parser = self.create_child(path.clone()); let _ = thread::spawn(move || { - let mut parser = - Parser::child_from_file(path, paths, bibliography, download_manager).unwrap(); - parser.set_config(config); - let document = parser.parse(); + let document = chid_parser.parse(); anchor_clone.write().unwrap().set_document(document); drop(wg); @@ -242,7 +187,8 @@ impl Parser { /// Imports a bibliography toml file fn import_bib(&mut self, path: PathBuf) -> ParseResult<()> { let f = File::open(path).map_err(|_| self.ctm.err())?; - self.document + self.options + .document .bibliography .read_bib_file(&mut BufReader::new(f)) .map_err(|_| self.ctm.err())?; @@ -256,8 +202,9 @@ impl Parser { } fn import_stylesheet(&mut self, path: PathBuf) -> ParseResult<()> { - self.document.stylesheets.push( - self.document + self.options.document.stylesheets.push( + self.options + .document .downloads .lock() .unwrap() @@ -272,7 +219,7 @@ impl Parser { let value = contents .parse::() .map_err(|_| self.ctm.err())?; - self.document.config.set_from_toml(&value); + self.options.document.config.set_from_toml(&value); Ok(()) } @@ -298,6 +245,7 @@ impl Parser { .and_then(|f| Some(f.to_str().unwrap().to_string())) { if let Some(Value::Array(ignore)) = self + .options .document .config .get_entry(IMP_IGNORE) @@ -345,7 +293,7 @@ impl Parser { /// parses the given text into a document pub fn parse(&mut self) -> Document { - self.document.path = if let Some(path) = &self.path { + self.options.document.path = if let Some(path) = &self.options.path { Some(path.canonicalize().unwrap().to_str().unwrap().to_string()) } else { None @@ -353,7 +301,7 @@ impl Parser { while !self.ctm.check_eof() { match self.parse_block() { - Ok(block) => self.document.add_element(block), + Ok(block) => self.options.document.add_element(block), Err(err) => { if self.ctm.check_eof() { break; @@ -366,7 +314,7 @@ impl Parser { let wg = self.wg.clone(); self.wg = WaitGroup::new(); - if !self.is_child { + if !self.options.is_child { for (path, file_type) in DEFAULT_IMPORTS { if self.transform_path(path.to_string()).exists() { self.import( @@ -377,19 +325,23 @@ impl Parser { } } wg.wait(); - if !self.is_child { + if !self.options.is_child { self.import_from_config(); } - self.document.post_process(); - let document = self.document.clone(); - self.document = Document::new(!self.is_child); + self.options.document.post_process(); + let document = std::mem::replace(&mut self.options.document, Document::new()); document } + pub fn get_paths(&self) -> Vec { + self.options.paths.lock().unwrap().clone() + } + /// Imports files from the configs import values fn import_from_config(&mut self) { if let Some(Value::Array(mut imp)) = self + .options .document .config .get_entry(IMP_STYLESHEETS) @@ -402,6 +354,7 @@ impl Parser { } } if let Some(Value::Array(mut imp)) = self + .options .document .config .get_entry(IMP_CONFIGS) @@ -413,6 +366,7 @@ impl Parser { } } if let Some(Value::Array(mut imp)) = self + .options .document .config .get_entry(IMP_BIBLIOGRAPHY) diff --git a/src/utils/downloads.rs b/src/utils/downloads.rs index dcc764c..f6e7661 100644 --- a/src/utils/downloads.rs +++ b/src/utils/downloads.rs @@ -1,6 +1,10 @@ use indicatif::{ProgressBar, ProgressStyle}; +use platform_dirs::{AppDirs, AppUI}; use rayon::prelude::*; +use std::collections::hash_map::DefaultHasher; +use std::fs; use std::fs::read; +use std::hash::{Hash, Hasher}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -8,6 +12,7 @@ use std::sync::{Arc, Mutex}; #[derive(Clone, Debug)] pub struct DownloadManager { downloads: Vec>>, + pub use_cache: bool, } impl DownloadManager { @@ -15,12 +20,15 @@ impl DownloadManager { pub fn new() -> Self { Self { downloads: Vec::new(), + use_cache: true, } } /// Adds a new pending download pub fn add_download(&mut self, path: String) -> Arc> { - let pending = Arc::new(Mutex::new(PendingDownload::new(path.clone()))); + let mut download = PendingDownload::new(path.clone()); + download.use_cache = self.use_cache; + let pending = Arc::new(Mutex::new(download)); self.downloads.push(Arc::clone(&pending)); log::debug!("Added download {}", path); @@ -51,11 +59,16 @@ impl DownloadManager { pub struct PendingDownload { pub(crate) path: String, pub(crate) data: Option>, + pub(crate) use_cache: bool, } impl PendingDownload { pub fn new(path: String) -> Self { - Self { path, data: None } + Self { + path, + data: None, + use_cache: true, + } } /// Downloads the file and writes the content to the content field @@ -66,10 +79,40 @@ impl PendingDownload { /// Reads the fiels content or downloads it if it doesn't exist in the filesystem fn read_content(&self) -> Option> { let path = PathBuf::from(&self.path); + if path.exists() { read(path).ok() + } else if let Some(contents) = self.read_from_cache() { + log::debug!("Read {} from cache.", self.path.clone()); + Some(contents) + } else { + if let Some(data) = self.download_content() { + self.store_to_cache(&data); + Some(data) + } else { + None + } + } + } + + /// Stores the data to a cache file to retrieve it later + fn store_to_cache(&self, data: &Vec) { + let cache_file = get_cached_path(PathBuf::from(&self.path)); + fs::write(&cache_file, data.clone()).unwrap_or_else(|_| { + log::warn!( + "Failed to write file to cache: {} -> {:?}", + self.path.clone(), + cache_file + ) + }); + } + + fn read_from_cache(&self) -> Option> { + let cache_path = get_cached_path(PathBuf::from(&self.path)); + if cache_path.exists() && self.use_cache { + read(cache_path).ok() } else { - self.download_content() + None } } @@ -82,3 +125,19 @@ impl PendingDownload { .map(|b| b.to_vec()) } } + +fn get_cached_path(path: PathBuf) -> PathBuf { + lazy_static::lazy_static! { + static ref APP_DIRS: AppDirs = AppDirs::new(Some("snekdown"), AppUI::CommandLine).unwrap(); + } + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + let file_name = PathBuf::from(format!("{:x}", hasher.finish())); + + if !APP_DIRS.cache_dir.is_dir() { + fs::create_dir(&APP_DIRS.cache_dir) + .unwrap_or_else(|_| log::warn!("Failed to create cache dir {:?}", APP_DIRS.cache_dir)) + } + + APP_DIRS.cache_dir.join(file_name) +} diff --git a/tests/parsing_tests.rs b/tests/parsing_tests.rs index a8d972c..65abcc9 100644 --- a/tests/parsing_tests.rs +++ b/tests/parsing_tests.rs @@ -13,11 +13,6 @@ macro_rules! count_block_elements { }; } -#[test] -fn it_inits() { - let _ = Parser::new("".to_string(), None); -} - #[test] fn it_parses_sections() { let document = parse!("# Section\n## Subsection\n# Section");