From 0c2c62e1572270ea4ccc4406417fec3e956abb4f Mon Sep 17 00:00:00 2001 From: trivernis Date: Sat, 10 Apr 2021 22:22:07 +0200 Subject: [PATCH] Add subcrate bot-coreutils Add subcrate bot-coreutils to move some utilities that don't depend too heavily on other crates and require heavy testing there. Signed-off-by: trivernis --- .github/workflows/build.yml | 8 +- Cargo.lock | 10 +++ Cargo.toml | 1 + bot-coreutils/.gitignore | 1 + bot-coreutils/Cargo.lock | 125 +++++++++++++++++++++++++++ bot-coreutils/Cargo.toml | 12 +++ bot-coreutils/src/lib.rs | 5 ++ bot-coreutils/src/tests/mod.rs | 2 + bot-coreutils/src/tests/url_tests.rs | 19 ++++ bot-coreutils/src/url.rs | 43 +++++++++ src/messages/sauce.rs | 2 +- src/utils/mod.rs | 11 --- 12 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 bot-coreutils/.gitignore create mode 100644 bot-coreutils/Cargo.lock create mode 100644 bot-coreutils/Cargo.toml create mode 100644 bot-coreutils/src/lib.rs create mode 100644 bot-coreutils/src/tests/mod.rs create mode 100644 bot-coreutils/src/tests/url_tests.rs create mode 100644 bot-coreutils/src/url.rs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfc9164..b80fe46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,5 +35,11 @@ jobs: - name: Build run: cargo build --verbose - - name: Run tests + - name: Test coreutils + run: cargo test --verbose --package bot-coreutils + + - name: Test database + run: cargo test --verbose --package bot-database + + - name: Test binary run: cargo test --verbose \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 312cff8..c4181bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,6 +178,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bot-coreutils" +version = "0.1.0" +dependencies = [ + "log 0.4.14", + "tokio", + "url", +] + [[package]] name = "bot-database" version = "0.1.0" @@ -2272,6 +2281,7 @@ name = "tobi-rs" version = "0.2.1" dependencies = [ "aspotify", + "bot-coreutils", "bot-database", "chrono", "chrono-tz", diff --git a/Cargo.toml b/Cargo.toml index 964fd49..8f6728c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" [dependencies] bot-database = {path="./bot-database"} +bot-coreutils = {path="./bot-coreutils"} serenity = "0.10.5" dotenv = "0.15.0" tokio = { version = "1.4.0", features = ["macros", "rt-multi-thread"] } diff --git a/bot-coreutils/.gitignore b/bot-coreutils/.gitignore new file mode 100644 index 0000000..1de5659 --- /dev/null +++ b/bot-coreutils/.gitignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/bot-coreutils/Cargo.lock b/bot-coreutils/Cargo.lock new file mode 100644 index 0000000..39d41b4 --- /dev/null +++ b/bot-coreutils/Cargo.lock @@ -0,0 +1,125 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bot-coreutils" +version = "0.1.0" +dependencies = [ + "log", + "tokio", + "url", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "idna" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project-lite" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" + +[[package]] +name = "tinyvec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134af885d758d645f0f0505c9a8b3f9bf8a348fd822e112ab5248138348f1722" +dependencies = [ + "autocfg", + "pin-project-lite", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] diff --git a/bot-coreutils/Cargo.toml b/bot-coreutils/Cargo.toml new file mode 100644 index 0000000..3444171 --- /dev/null +++ b/bot-coreutils/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "bot-coreutils" +version = "0.1.0" +authors = ["trivernis "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = "1.4.0" +log = "0.4.14" +url = "2.2.1" \ No newline at end of file diff --git a/bot-coreutils/src/lib.rs b/bot-coreutils/src/lib.rs new file mode 100644 index 0000000..7f750f1 --- /dev/null +++ b/bot-coreutils/src/lib.rs @@ -0,0 +1,5 @@ +#[cfg(test)] +mod tests; + +/// Utilities to quickly check strings that represent urls +pub mod url; diff --git a/bot-coreutils/src/tests/mod.rs b/bot-coreutils/src/tests/mod.rs new file mode 100644 index 0000000..26d5466 --- /dev/null +++ b/bot-coreutils/src/tests/mod.rs @@ -0,0 +1,2 @@ +#[cfg(test)] +mod url_tests; \ No newline at end of file diff --git a/bot-coreutils/src/tests/url_tests.rs b/bot-coreutils/src/tests/url_tests.rs new file mode 100644 index 0000000..2f3cac4 --- /dev/null +++ b/bot-coreutils/src/tests/url_tests.rs @@ -0,0 +1,19 @@ +use crate::url::*; + +#[test] +fn it_returns_the_domain_name() { + assert_eq!( + get_domain_for_url("https://domain.com/sub/sub"), + Some("domain.com".to_string()) + ); + assert_eq!( + get_domain_for_url("other-domain.com"), + Some("other-domain.com".to_string()) + ); + assert_eq!(get_domain_for_url("Invalid URL"), None); + assert_eq!(get_domain_for_url("file:////what/a/file.txt"), None); + assert_eq!( + get_domain_for_url("https://www.domain.com/sub",), + Some("domain.com".to_string()) + ); +} diff --git a/bot-coreutils/src/url.rs b/bot-coreutils/src/url.rs new file mode 100644 index 0000000..6c9f23b --- /dev/null +++ b/bot-coreutils/src/url.rs @@ -0,0 +1,43 @@ +pub use url::*; + +static PROTOCOL_HTTP: &str = "http://"; +static PROTOCOL_HTTPS: &str = "https://"; +static PROTOCOL_FILE: &str = "file:////"; +static PROTOCOL_DATA: &str = "data:"; +static PROTOCOLS: &[&str] = &[PROTOCOL_HTTP, PROTOCOL_HTTPS, PROTOCOL_FILE, PROTOCOL_DATA]; + +/// Adds the protocol in front of the url if it is missing from the input +fn add_missing_protocol(url_str: &str) -> String { + for protocol in PROTOCOLS { + if url_str.starts_with(protocol) { + return url_str.to_string(); + } + } + + format!("{}{}", PROTOCOL_HTTPS, url_str) +} + +/// Parses the given url into the url representation +/// Allows for fuzzier input than the original method. If no protocol is provided, +/// it assumes https. +#[inline] +pub fn parse_url(url_str: &str) -> Result { + let url_str = add_missing_protocol(url_str); + Url::parse(&url_str) +} + +/// Returns the domain for a given url string +/// Example +/// ``` +/// use bot_coreutils::url::get_domain_for_url; +/// +/// assert_eq!(get_domain_for_url("https://reddit.com/r/anime"), Some("reddit.com".to_string())); +/// assert_eq!(get_domain_for_url("reddit.com"), Some("reddit.com".to_string())); +/// assert_eq!(get_domain_for_url("invalid url"), None); +/// ``` +pub fn get_domain_for_url(url_str: &str) -> Option { + let url = parse_url(url_str).ok()?; + let domain = url.domain()?; + + Some(domain.trim_start_matches("www.").to_string()) +} diff --git a/src/messages/sauce.rs b/src/messages/sauce.rs index 78ffa44..0cb19d9 100644 --- a/src/messages/sauce.rs +++ b/src/messages/sauce.rs @@ -1,5 +1,5 @@ use crate::utils::error::BotResult; -use crate::utils::get_domain_for_url; +use bot_coreutils::url::get_domain_for_url; use sauce_api::{SauceItem, SauceResult}; use serenity::builder::CreateMessage; use serenity::{model::channel::Message, prelude::*}; diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 6fe164e..683bac3 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -39,17 +39,6 @@ pub async fn get_previous_message_or_reply( Ok(referenced) } -/// Returns the domain for a given url -pub fn get_domain_for_url(url: &str) -> Option { - lazy_static::lazy_static! { - static ref DOMAIN_REGEX: Regex = Regex::new(r"^(https?://)?(www\.)?((\w+\.)+\w+).*$").unwrap(); - } - - let captures = DOMAIN_REGEX.captures(url)?; - - captures.get(3).map(|c| c.as_str().to_string()) -} - /// Returns the file for a given domain pub fn get_file_name_for_domain(url: &str) -> Option { lazy_static::lazy_static! {