Add subcrate bot-coreutils
Add subcrate bot-coreutils to move some utilities that don't depend too heavily on other crates and require heavy testing there. Signed-off-by: trivernis <trivernis@protonmail.com>pull/7/head
parent
d54361deca
commit
0c2c62e157
GPG Key ID:
DFFFCC2C7A02DB45
@ -0,0 +1 @@
|
||||
target
|
||||
@ -0,0 +1,125 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
||||
|
||||
[[package]]
|
||||
name = "bot-coreutils"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tokio",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
|
||||
dependencies = [
|
||||
"matches",
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21"
|
||||
dependencies = [
|
||||
"matches",
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matches"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905"
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec_macros"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "134af885d758d645f0f0505c9a8b3f9bf8a348fd822e112ab5248138348f1722"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
|
||||
dependencies = [
|
||||
"matches",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
"matches",
|
||||
"percent-encoding",
|
||||
]
|
||||
@ -0,0 +1,12 @@
|
||||
[package]
|
||||
name = "bot-coreutils"
|
||||
version = "0.1.0"
|
||||
authors = ["trivernis <trivernis@protonmail.com>"]
|
||||
edition = "2018"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
tokio = "1.4.0"
|
||||
log = "0.4.14"
|
||||
url = "2.2.1"
|
||||
@ -0,0 +1,5 @@
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
/// Utilities to quickly check strings that represent urls
|
||||
pub mod url;
|
||||
@ -0,0 +1,2 @@
|
||||
#[cfg(test)]
|
||||
mod url_tests;
|
||||
@ -0,0 +1,19 @@
|
||||
use crate::url::*;
|
||||
|
||||
#[test]
|
||||
fn it_returns_the_domain_name() {
|
||||
assert_eq!(
|
||||
get_domain_for_url("https://domain.com/sub/sub"),
|
||||
Some("domain.com".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
get_domain_for_url("other-domain.com"),
|
||||
Some("other-domain.com".to_string())
|
||||
);
|
||||
assert_eq!(get_domain_for_url("Invalid URL"), None);
|
||||
assert_eq!(get_domain_for_url("file:////what/a/file.txt"), None);
|
||||
assert_eq!(
|
||||
get_domain_for_url("https://www.domain.com/sub",),
|
||||
Some("domain.com".to_string())
|
||||
);
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
pub use url::*;
|
||||
|
||||
static PROTOCOL_HTTP: &str = "http://";
|
||||
static PROTOCOL_HTTPS: &str = "https://";
|
||||
static PROTOCOL_FILE: &str = "file:////";
|
||||
static PROTOCOL_DATA: &str = "data:";
|
||||
static PROTOCOLS: &[&str] = &[PROTOCOL_HTTP, PROTOCOL_HTTPS, PROTOCOL_FILE, PROTOCOL_DATA];
|
||||
|
||||
/// Adds the protocol in front of the url if it is missing from the input
|
||||
fn add_missing_protocol(url_str: &str) -> String {
|
||||
for protocol in PROTOCOLS {
|
||||
if url_str.starts_with(protocol) {
|
||||
return url_str.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
format!("{}{}", PROTOCOL_HTTPS, url_str)
|
||||
}
|
||||
|
||||
/// Parses the given url into the url representation
|
||||
/// Allows for fuzzier input than the original method. If no protocol is provided,
|
||||
/// it assumes https.
|
||||
#[inline]
|
||||
pub fn parse_url(url_str: &str) -> Result<Url, url::ParseError> {
|
||||
let url_str = add_missing_protocol(url_str);
|
||||
Url::parse(&url_str)
|
||||
}
|
||||
|
||||
/// Returns the domain for a given url string
|
||||
/// Example
|
||||
/// ```
|
||||
/// use bot_coreutils::url::get_domain_for_url;
|
||||
///
|
||||
/// assert_eq!(get_domain_for_url("https://reddit.com/r/anime"), Some("reddit.com".to_string()));
|
||||
/// assert_eq!(get_domain_for_url("reddit.com"), Some("reddit.com".to_string()));
|
||||
/// assert_eq!(get_domain_for_url("invalid url"), None);
|
||||
/// ```
|
||||
pub fn get_domain_for_url(url_str: &str) -> Option<String> {
|
||||
let url = parse_url(url_str).ok()?;
|
||||
let domain = url.domain()?;
|
||||
|
||||
Some(domain.trim_start_matches("www.").to_string())
|
||||
}
|
||||
Loading…
Reference in New Issue