Trivernis
/
hydrus-utils
mirror of https://github.com/Trivernis/hydrus-utils.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move api options to config file and improve reddit file parsing

Browse Source 
Signed-off-by: trivernis <trivernis@protonmail.com>
main
trivernis 2 years ago
parent c918e9ef3a
commit 7e1e1d9de1
Signed by: Trivernis
GPG Key ID: DFFFCC2C7A02DB45
11 changed files with 503 additions and 19 deletions
Show Stats Download Patch File Download Diff File

1
.gitignore vendored
Unescape Escape View File

@ -1,3 +1,4 @@
/target
reddit-urls.txt
reddt-urls.txt
urls.txt

357
Cargo.lock generated
Unescape Escape View File

@ -17,6 +17,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "aho-corasick"
version = "0.7.18"
@ -44,6 +55,17 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "async-trait"
version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "atty"
version = "0.2.14"
@ -106,6 +128,15 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "block-buffer"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
dependencies = [
"generic-array",
]
[[package]]
name = "bumpalo"
version = "3.10.0"
@ -216,6 +247,52 @@ dependencies = [
"bitflags",
]
[[package]]
name = "color-eyre"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204"
dependencies = [
"backtrace",
"color-spantrace",
"eyre",
"indenter",
"once_cell",
"owo-colors",
"tracing-error",
]
[[package]]
name = "color-spantrace"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce"
dependencies = [
"once_cell",
"owo-colors",
"tracing-core",
"tracing-error",
]
[[package]]
name = "config"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11f1667b8320afa80d69d8bbe40830df2c8a06003d86f73d8e003b2c48df416d"
dependencies = [
"async-trait",
"json5",
"lazy_static",
"nom",
"pathdiff",
"ron",
"rust-ini",
"serde",
"serde_json",
"toml",
"yaml-rust",
]
[[package]]
name = "cookie"
version = "0.12.0"
@ -260,6 +337,15 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "cpufeatures"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc948ebb96241bb40ab73effeb80d9f93afaad49359d159a5e61be51619fe813"
dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.3.2"
@ -317,6 +403,52 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "digest"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "directories"
version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
"winapi 0.3.9",
]
[[package]]
name = "dlv-list"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
[[package]]
name = "dtoa"
version = "0.4.8"
@ -338,6 +470,16 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "eyre"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb"
dependencies = [
"indenter",
"once_cell",
]
[[package]]
name = "failure"
version = "0.1.8"
@ -487,6 +629,27 @@ dependencies = [
"pin-utils",
]
[[package]]
name = "generic-array"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
dependencies = [
"cfg-if 1.0.0",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
name = "gimli"
version = "0.26.2"
@ -535,6 +698,9 @@ name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash",
]
[[package]]
name = "heck"
@ -630,6 +796,9 @@ name = "hydrus-utils"
version = "0.3.0"
dependencies = [
"clap",
"color-eyre",
"config",
"directories",
"hydrus-api",
"pixiv-rs",
"reqwest 0.11.11",
@ -758,6 +927,12 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "indenter"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
[[package]]
name = "indexmap"
version = "1.9.1"
@ -813,6 +988,17 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "json5"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1"
dependencies = [
"pest",
"pest_derive",
"serde",
]
[[package]]
name = "kernel32-sys"
version = "0.2.2"
@ -835,6 +1021,12 @@ version = "0.2.131"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04c3b4822ccebfa39c02fc03d1534441b22ead323fa0f48bb7ddd8e6ba076a40"
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "lock_api"
version = "0.3.4"
@ -905,6 +1097,12 @@ dependencies = [
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.5.3"
@ -986,6 +1184,16 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "nom"
version = "7.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "num-integer"
version = "0.1.45"
@ -1075,12 +1283,28 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "ordered-multimap"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
dependencies = [
"dlv-list",
"hashbrown",
]
[[package]]
name = "os_str_bytes"
version = "6.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4"
[[package]]
name = "owo-colors"
version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "parking_lot"
version = "0.9.0"
@ -1107,6 +1331,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "pathdiff"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
[[package]]
name = "percent-encoding"
version = "1.0.1"
@ -1119,6 +1349,50 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pest"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b0560d531d1febc25a3c9398a62a71256c0178f2e3443baedd9ad4bb8c9deb4"
dependencies = [
"thiserror",
"ucd-trie",
]
[[package]]
name = "pest_derive"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "905708f7f674518498c1f8d644481440f476d39ca6ecae83319bba7c6c12da91"
dependencies = [
"pest",
"pest_generator",
]
[[package]]
name = "pest_generator"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5803d8284a629cc999094ecd630f55e91b561a1d1ba75e233b00ae13b91a69ad"
dependencies = [
"pest",
"pest_meta",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pest_meta"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1538eb784f07615c6d9a8ab061089c6c54a344c5b4301db51990ca1c241e8c04"
dependencies = [
"once_cell",
"pest",
"sha-1",
]
[[package]]
name = "pin-project-lite"
version = "0.2.9"
@ -1343,6 +1617,17 @@ dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom",
"redox_syscall 0.2.16",
"thiserror",
]
[[package]]
name = "regex"
version = "1.6.0"
@ -1449,6 +1734,27 @@ dependencies = [
"winreg 0.10.1",
]
[[package]]
name = "ron"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a"
dependencies = [
"base64 0.13.0",
"bitflags",
"serde",
]
[[package]]
name = "rust-ini"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
dependencies = [
"cfg-if 1.0.0",
"ordered-multimap",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
@ -1592,6 +1898,17 @@ dependencies = [
"serde",
]
[[package]]
name = "sha-1"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f"
dependencies = [
"cfg-if 1.0.0",
"cpufeatures",
"digest",
]
[[package]]
name = "sharded-slab"
version = "0.1.4"
@ -1953,6 +2270,15 @@ dependencies = [
"tracing",
]
[[package]]
name = "toml"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
dependencies = [
"serde",
]
[[package]]
name = "tower-service"
version = "0.3.2"
@ -1992,6 +2318,16 @@ dependencies = [
"valuable",
]
[[package]]
name = "tracing-error"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
dependencies = [
"tracing",
"tracing-subscriber",
]
[[package]]
name = "tracing-log"
version = "0.1.3"
@ -2036,6 +2372,18 @@ dependencies = [
"cfg-if 0.1.10",
]
[[package]]
name = "typenum"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "ucd-trie"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89570599c4fe5585de2b388aab47e99f7fa4e9238a1399f707a02e356058141c"
[[package]]
name = "unicase"
version = "2.6.0"
@ -2344,3 +2692,12 @@ dependencies = [
"winapi 0.2.8",
"winapi-build",
]
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]

3
Cargo.toml
Unescape Escape View File

@ -22,6 +22,9 @@ clap = { version = "3.2.17", features = ["derive", "env"] }
serde = { version = "1.0.143", features = ["derive"] }
reqwest = { version = "0.11.11", features = ["json"] }
serde_json = "1.0.83"
config = "0.13.2"
directories = "4.0.1"
color-eyre = "0.6.2"
[dependencies.tokio]
version = "1.20.1"

12
src/args.rs
Unescape Escape View File

@ -7,14 +7,6 @@ use clap::{Parser, Subcommand};
pub struct Args {
#[clap(subcommand)]
pub subcommand: Command,
/// The hydrus client api key
#[clap(long, env)]
pub hydrus_key: String,
/// The url to the hydrus client api
#[clap(long, default_value = "http://127.0.0.1:45869", env)]
pub hydrus_url: String,
}
#[derive(Subcommand, Clone, Debug)]
@ -34,10 +26,6 @@ pub enum Command {
#[derive(Parser, Debug, Clone)]
pub struct LookupOptions {
/// The saucenao api key
#[clap(long, env)]
pub saucenao_key: String,
/// The tag service the tags will be assigned to
#[clap(long, default_value = "my tags")]
pub tag_service: String,

14
src/assets/config.toml
Unescape Escape View File

@ -0,0 +1,14 @@
# Hydrus utils default config
[hydrus]
api_url = "http://127.0.0.1:45869"
# api_key = "<API KEY>"
# # Settings for saucenao file lookup
# [saucenao]
# api_key = "<API KEY>"
# # Settings for fetching media from twitter urls
# [twitter]
# consumer_key = "<CONSUMER KEY>"
# consumer_secret = "<CONSUMER SECRET>"

53
src/config.rs
Unescape Escape View File

@ -0,0 +1,53 @@
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use crate::{error::Result, utils::get_config_dir};
use std::fs;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Config {
pub hydrus: HydrusConfig,
pub saucenao: Option<SauceNaoConfig>,
pub twitter: Option<TwitterConfig>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct HydrusConfig {
pub api_url: String,
pub api_key: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SauceNaoConfig {
pub api_key: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TwitterConfig {
pub consumer_key: String,
pub consumer_secret: String,
}
impl Config {
pub fn read() -> Result<Self> {
let config_dir = get_config_dir()?;
let config_file_path = config_dir.join(PathBuf::from("config.toml"));
if !config_file_path.exists() {
fs::write(&config_file_path, include_str!("assets/config.toml"))?;
}
let settings = config::Config::builder()
.add_source(config::File::with_name(config_file_path.to_str().unwrap()))
.build()?;
tracing::debug!("Config is {settings:?}");
Ok(settings.try_deserialize()?)
}
/// Returns the saucenao configuratio or panics if nothing is configured
pub fn into_saucenao(self) -> SauceNaoConfig {
self.saucenao
.expect("No saucenao key configured. Please add one to the config file.")
}
}

3
src/error.rs
Unescape Escape View File

@ -21,6 +21,9 @@ pub enum Error {
#[error(transparent)]
Reqwest(#[from] reqwest::Error),
#[error("Error in config {0}")]
Config(#[from] config::ConfigError),
}
impl From<RustNaoError> for Error {

30
src/main.rs
Unescape Escape View File

@ -1,8 +1,11 @@
mod args;
mod config;
mod error;
mod operations;
pub mod utils;
use crate::config::Config;
use crate::config::SauceNaoConfig;
use crate::error::Result;
use crate::operations::find_and_send_tags::find_and_send_tags;
use crate::operations::find_and_send_urls::find_and_send_urls;
@ -24,14 +27,20 @@ use tracing_subscriber::EnvFilter;
#[tokio::main(flavor = "current_thread")]
async fn main() {
color_eyre::install().unwrap();
init_logger();
let args: Args = Args::parse();
let config = Config::read().expect("Failed to read configuration");
tracing::debug!("args: {args:?}");
let hydrus = Hydrus::new(Client::new(&args.hydrus_url, &args.hydrus_key));
let hydrus = Hydrus::new(Client::new(&config.hydrus.api_url, &config.hydrus.api_key));
match args.subcommand {
Command::FindAndSendUrl(opt) => send_tags_or_urls(opt, hydrus, true).await,
Command::FindAndSendTags(opt) => send_tags_or_urls(opt, hydrus, false).await,
Command::FindAndSendUrl(opt) => {
send_tags_or_urls(opt, config.into_saucenao(), hydrus, true).await
}
Command::FindAndSendTags(opt) => {
send_tags_or_urls(opt, config.into_saucenao(), hydrus, false).await
}
Command::ImportRedditPosts(opt) => import_reddit_posts(opt, hydrus).await,
}
.expect("Failed to send tags or urls");
@ -51,11 +60,17 @@ fn init_logger() {
.init();
}
async fn send_tags_or_urls(opt: LookupOptions, hydrus: Hydrus, send_urls: bool) -> Result<()> {
#[tracing::instrument(level = "debug", skip(hydrus))]
async fn send_tags_or_urls(
opt: LookupOptions,
saucenao_cfg: SauceNaoConfig,
hydrus: Hydrus,
send_urls: bool,
) -> Result<()> {
let pixiv = PixivClient::new();
let handler = HandlerBuilder::new()
.api_key(&opt.saucenao_key)
.api_key(&saucenao_cfg.api_key)
.min_similarity(80)
.db(Handler::PIXIV)
.build();
@ -94,6 +109,7 @@ async fn send_tags_or_urls(opt: LookupOptions, hydrus: Hydrus, send_urls: bool)
Ok(())
}
#[tracing::instrument(level = "debug", skip(hydrus))]
async fn import_reddit_posts(opt: ImportRedditOptions, hydrus: Hydrus) -> Result<()> {
let mut urls = Vec::new();
@ -103,7 +119,9 @@ async fn import_reddit_posts(opt: ImportRedditOptions, hydrus: Hydrus) -> Result
let mut lines = reader.lines();
while let Some(line) = lines.next_line().await? {
urls.push(line);
if line.len() > 0 {
urls.push(line);
}
}
} else if let Some(args_urls) = opt.urls {
urls = args_urls;

1
src/operations/find_and_send_reddit_posts.rs
Unescape Escape View File

@ -3,6 +3,7 @@ use hydrus_api::Hydrus;
use crate::error::Result;
use crate::utils::reddit::get_post_images;
#[tracing::instrument(level = "debug", skip(hydrus))]
pub async fn find_and_send_reddit_posts(hydrus: &Hydrus, post_urls: Vec<String>) -> Result<()> {
let total_posts = post_urls.len();

19
src/utils/mod.rs
Unescape Escape View File

@ -1,2 +1,21 @@
pub mod pixiv;
pub mod reddit;
use crate::error::Result;
use directories::ProjectDirs;
use std::{fs, path::PathBuf};
pub fn get_project_dirs() -> ProjectDirs {
ProjectDirs::from("net", "trivernis", "hydrus-utils")
.expect("Could not create application directories")
}
pub fn get_config_dir() -> Result<PathBuf> {
let dirs = get_project_dirs();
let config_dir = dirs.config_dir();
if !config_dir.exists() {
fs::create_dir_all(config_dir)?;
}
Ok(PathBuf::from(config_dir))
}

29
src/utils/reddit.rs
Unescape Escape View File

@ -2,8 +2,10 @@
use std::collections::HashMap;
use crate::Result;
use reqwest::{redirect::Policy, StatusCode};
use serde::Deserialize;
use serde_json::Value;
use std::fmt::Debug;
#[derive(Deserialize)]
#[serde(tag = "kind", content = "data")]
@ -50,7 +52,8 @@ struct GalleryItem {
}
/// Returns all images associated with a post
pub async fn get_post_images<S: AsRef<str>>(post_url: S) -> Result<Vec<String>> {
#[tracing::instrument(level = "debug")]
pub async fn get_post_images<S: AsRef<str> + Debug>(post_url: S) -> Result<Vec<String>> {
let post_data = get_post(post_url.as_ref()).await?;
if let Some(gallery_data) = post_data.gallery_data {
@ -68,7 +71,18 @@ pub async fn get_post_images<S: AsRef<str>>(post_url: S) -> Result<Vec<String>>
}
}
#[tracing::instrument(level = "debug")]
async fn get_post(url: &str) -> Result<T3Data> {
let mut url = resolve_redirects(url).await?;
// url cleanup
// add trailing slash and remove path params
if !url.ends_with('/') {
if let Some((left, right)) = url.rsplit_once('?') {
url = left.to_string();
}
url.push('/');
}
let mut response: Vec<DataEntry> = reqwest::get(format!("{}.json", url)).await?.json().await?;
response.reverse();
let first_entry = response.pop().unwrap();
@ -84,6 +98,19 @@ async fn get_post(url: &str) -> Result<T3Data> {
}
}
/// Resolves reddit redirects
#[tracing::instrument(level = "debug")]
async fn resolve_redirects(url: &str) -> Result<String> {
let client = reqwest::Client::builder()
.redirect(Policy::none())
.build()?;
let response = client.get(url).send().await?;
if let Some(location) = response.headers().get("location") {
return Ok(location.to_str().unwrap().to_string());
}
Ok(url.to_string())
}
#[cfg(test)]
mod test {
#[tokio::test]

Write Preview
Loading…
Cancel
Save