chore: Release hydrus-utils version 0.6.0

Add tags command to add tags to files and update hydrus api
Fix reddit wanting a user agent now
12 changed files with 345 additions and 83 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -54,7 +54,7 @@ checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -220,7 +220,7 @@ dependencies = [
 "proc-macro-error",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -457,7 +457,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "scratch",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -474,7 +474,7 @@ checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -487,7 +487,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "rustc_version 0.4.0",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -550,7 +550,7 @@ dependencies = [
 "base64 0.13.1",
 "chrono",
 "derive_more",
- "futures 0.3.26",
+ "futures 0.3.28",
 "hmac",
 "hyper 0.14.24",
 "hyper-tls 0.5.0",
@ -632,7 +632,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 "synstructure",
 ]

@ -728,9 +728,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"

 [[package]]
 name = "futures"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
+checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
 dependencies = [
 "futures-channel",
 "futures-core",
@ -743,9 +743,9 @@ dependencies = [

 [[package]]
 name = "futures-channel"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
+checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
 dependencies = [
 "futures-core",
 "futures-sink",
@ -753,9 +753,9 @@ dependencies = [

 [[package]]
 name = "futures-core"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
+checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"

 [[package]]
 name = "futures-cpupool"
@ -769,9 +769,9 @@ dependencies = [

 [[package]]
 name = "futures-executor"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
+checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
 dependencies = [
 "futures-core",
 "futures-task",
@ -780,38 +780,38 @@ dependencies = [

 [[package]]
 name = "futures-io"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"
+checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"

 [[package]]
 name = "futures-macro"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
+checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.23",
 ]

 [[package]]
 name = "futures-sink"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
+checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"

 [[package]]
 name = "futures-task"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
+checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"

 [[package]]
 name = "futures-util"
-version = "0.3.26"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
+checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
 dependencies = [
 "futures-channel",
 "futures-core",
@ -988,9 +988,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"

 [[package]]
 name = "hydrus-api"
-version = "0.9.3"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3ee4a01380e02f9bf73d14e87021c756bc06d9838662a6bc27c786bba3b26c0"
+checksum = "4d751951817b71e4d86ffd345fbfa0113fe8786c7696d49b8820d1af37cc9634"
 dependencies = [
 "bytes 1.4.0",
 "chrono",
@ -1005,7 +1005,7 @@ dependencies = [

 [[package]]
 name = "hydrus-utils"
-version = "0.5.2"
+version = "0.6.0"
 dependencies = [
 "clap",
 "color-eyre",
@ -1013,6 +1013,7 @@ dependencies = [
 "directories",
 "egg-mode",
 "fakeit",
+ "futures 0.3.28",
 "hydrus-api",
 "lazy-regex",
 "pixiv-rs",
@ -1287,7 +1288,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "regex",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -1582,7 +1583,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -1706,7 +1707,7 @@ dependencies = [
 "pest_meta",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -1764,7 +1765,7 @@ dependencies = [
 "proc-macro-error-attr",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 "version_check",
 ]

@ -1781,9 +1782,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.51"
+version = "1.0.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
+checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
 dependencies = [
 "unicode-ident",
 ]
@ -1800,9 +1801,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.23"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
 dependencies = [
 "proc-macro2",
 ]
@ -2267,7 +2268,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -2413,6 +2414,17 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "syn"
+version = "2.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "synstructure"
 version = "0.12.6"
@ -2421,7 +2433,7 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 "unicode-xid",
 ]

@ -2475,7 +2487,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -2600,7 +2612,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -2734,7 +2746,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 ]

 [[package]]
@ -2967,7 +2979,7 @@ dependencies = [
 "once_cell",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 "wasm-bindgen-shared",
 ]

@ -3001,7 +3013,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.107",
 "wasm-bindgen-backend",
 "wasm-bindgen-shared",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@ -1,6 +1,6 @@
 [package]
 name = "hydrus-utils"
-version = "0.5.2"
+version = "0.6.0"
 edition = "2021"
 license = "Apache-2.0"
 description = "Automatically tag hydrus file by using pixiv and saucenao"
@ -12,7 +12,7 @@ repository = "https://github.com/Trivernis/hydrus-pixiv-tagger"

 [dependencies]
 pixiv-rs = "0.1.0"
-hydrus-api = { version = "0.9.3", default-features = false, features = ["json"] }
+hydrus-api = { version = "0.10.2", default-features = false, features = ["json"] }
 rustnao = "0.2.1"
 tempdir = "0.3.7"
 thiserror = "1.0.38"
@ -28,6 +28,7 @@ color-eyre = "0.6.2"
 egg-mode = "0.16.1"
 lazy-regex = "2.4.1"
 fakeit = "1.1.1"
+futures = "0.3.28"

 [dependencies.tokio]
 version = "1.25.0"
--- a/src/args.rs
+++ b/src/args.rs
@ -23,9 +23,17 @@ pub enum Command {
    #[clap(name = "import-reddit-posts")]
    ImportRedditPosts(ImportUrlsOptions),

+    /// Looks up and imports fedi posts
+    #[clap(name = "import-fedi-posts")]
+    ImportFediPosts(ImportUrlsOptions),
+
    /// Looks up a list of urls and imports media found for them
    #[clap(name = "import-urls")]
    ImportUrls(ImportUrlsOptions),
+
+    /// Tag a file with a given identifier. The identifier is sent via stdin
+    #[clap(name = "tag")]
+    Tag(TagOptions),
 }

 #[derive(Parser, Debug, Clone)]
@ -54,3 +62,18 @@ pub struct ImportUrlsOptions {
    #[clap(short, long)]
    pub urls: Option<Vec<String>>,
 }
+
+#[derive(Parser, Debug, Clone)]
+pub struct TagOptions {
+    /// The tag service the tags will be assigned to
+    #[clap(long, default_value = "my tags")]
+    pub tag_service: String,
+
+    /// A list of file hashes
+    #[clap(long)]
+    pub files: Vec<String>,
+
+    /// The tags to assign
+    #[clap(short, long)]
+    pub tags: Vec<String>,
+}
--- a/src/main.rs
+++ b/src/main.rs
@ -7,10 +7,13 @@ pub mod utils;
 use crate::config::Config;
 use crate::config::SauceNaoConfig;
 use crate::error::Result;
+use crate::operations::find_and_send_fedi_posts::find_and_send_fedi_posts;
 use crate::operations::find_and_send_tags::find_and_send_tags;
 use crate::operations::find_and_send_urls::find_and_send_urls;
 use args::*;
 use clap::Parser;
+use hydrus_api::api_core::common::FileIdentifier;
+use hydrus_api::api_core::endpoints::adding_tags::TagAction;
 use hydrus_api::wrapper::service::ServiceName;
 use hydrus_api::wrapper::tag::Tag;
 use hydrus_api::{Client, Hydrus};
@ -44,7 +47,9 @@ async fn main() {
            send_tags_or_urls(opt, config.into_saucenao(), hydrus, false).await
        }
        Command::ImportRedditPosts(opt) => import_reddit_posts(opt, hydrus).await,
+        Command::ImportFediPosts(opt) => import_fedi_posts(opt, hydrus).await,
        Command::ImportUrls(opt) => import_urls(opt, hydrus).await,
+        Command::Tag(opt) => tag_files(opt, hydrus).await,
    }
    .expect("Failed to send tags or urls");
 }
@ -87,6 +92,7 @@ async fn send_tags_or_urls(

    let sleep_duration = Duration::from_secs(6);
    let total_files = files.len();
+    let service_key = hydrus.get_service_key(service.into()).await?;

    for (i, mut file) in files.into_iter().enumerate() {
        let start = Instant::now();
@ -99,7 +105,7 @@ async fn send_tags_or_urls(
                opt.finish_tag.as_ref(),
                &handler,
                &pixiv,
-                &service,
+                &service_key,
                &tmpdir,
                &mut file,
            )
@ -107,7 +113,7 @@ async fn send_tags_or_urls(
        }
        let elapsed = start.elapsed();

-        if elapsed.as_secs() < 8 {
+        if elapsed.as_secs() < 8 && sleep_duration > elapsed {
            tokio::time::sleep(sleep_duration - elapsed).await; // rate limit of 6# / 30s
        }
    }
@ -121,14 +127,22 @@ async fn import_reddit_posts(opt: ImportUrlsOptions, hydrus: Hydrus) -> Result<(
    find_and_send_reddit_posts(&hydrus, urls).await
 }

+#[tracing::instrument(level = "debug", skip(hydrus))]
+async fn import_fedi_posts(opt: ImportUrlsOptions, hydrus: Hydrus) -> Result<()> {
+    let urls = get_urls_from_args(opt).await?;
+    find_and_send_fedi_posts(&hydrus, urls).await
+}
+
 async fn import_urls(opt: ImportUrlsOptions, hydrus: Hydrus) -> Result<()> {
    let urls = get_urls_from_args(opt).await?;
    let mut reddit_urls = Vec::new();
+    let mut fedi_urls = Vec::new();
    let mut unknown_urls = Vec::new();

    for url in urls {
-        match find_url_type(&url) {
+        match find_url_type(&url).await {
            UrlType::Reddit => reddit_urls.push(url),
+            UrlType::Fedi => fedi_urls.push(url),
            UrlType::Other => {
                tracing::warn!("Unknown url type {url}");
                unknown_urls.push(url)
@ -137,6 +151,7 @@ async fn import_urls(opt: ImportUrlsOptions, hydrus: Hydrus) -> Result<()> {
    }
    tracing::info!("Importing reddit posts...");
    find_and_send_reddit_posts(&hydrus, reddit_urls).await?;
+    find_and_send_fedi_posts(&hydrus, fedi_urls).await?;

    tracing::info!("Importing unknown urls...");

@ -168,3 +183,19 @@ async fn get_urls_from_args(opt: ImportUrlsOptions) -> Result<Vec<String>> {
    }
    Ok(urls)
 }
+
+async fn tag_files(opt: TagOptions, hydrus: Hydrus) -> Result<()> {
+    let tags = opt.tags.into_iter().map(Tag::from).collect::<Vec<_>>();
+    let service_key = hydrus
+        .get_service_key(ServiceName(opt.tag_service).into())
+        .await?;
+
+    for file in opt.files {
+        hydrus
+            .file(FileIdentifier::hash(file))
+            .await?
+            .add_tags(service_key.to_owned(), tags.clone())
+            .await?;
+    }
+    Ok(())
+}
--- a/src/operations/find_and_send_fedi_posts.rs
+++ b/src/operations/find_and_send_fedi_posts.rs
@ -0,0 +1,35 @@
+use hydrus_api::Hydrus;
+
+use crate::error::Result;
+use crate::utils::fedi::get_post_images;
+
+#[tracing::instrument(level = "debug", skip(hydrus))]
+pub async fn find_and_send_fedi_posts(hydrus: &Hydrus, post_urls: Vec<String>) -> Result<()> {
+    let total_posts = post_urls.len();
+
+    for (index, post) in post_urls.into_iter().enumerate() {
+        tracing::info!("Importing post {} of {}", index + 1, total_posts);
+        if let Err(e) = import_post(&post, hydrus).await {
+            tracing::error!("Failed to import {}: {}", post, e);
+        }
+    }
+
+    Ok(())
+}
+
+#[tracing::instrument(level = "debug", skip(hydrus))]
+async fn import_post(post_url: &str, hydrus: &Hydrus) -> Result<()> {
+    tracing::debug!("Post {}", post_url);
+    let images = get_post_images(post_url).await?;
+    tracing::info!("Found {} images for post {}", images.len(), post_url);
+
+    for url in images {
+        let mut entry = hydrus.import().url(url).run().await?;
+        let files = entry.files().await?;
+
+        for mut file in files {
+            file.associate_urls(vec![post_url.to_string()]).await?;
+        }
+    }
+    Ok(())
+}
--- a/src/operations/find_and_send_reddit_posts.rs
+++ b/src/operations/find_and_send_reddit_posts.rs
@ -1,35 +1,76 @@
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
 use hydrus_api::Hydrus;
+use tokio::sync::Semaphore;

 use crate::error::Result;
 use crate::utils::reddit::get_post_images;
+use futures::future;

 #[tracing::instrument(level = "debug", skip(hydrus))]
 pub async fn find_and_send_reddit_posts(hydrus: &Hydrus, post_urls: Vec<String>) -> Result<()> {
    let total_posts = post_urls.len();
+    let mut posts_with_img = Vec::new();
+
+    tracing::info!("Retrieving post data...");
+    let counter = Arc::new(AtomicUsize::new(1));
+
+    let sem = Arc::new(Semaphore::new(2));
+    let post_results = future::join_all(post_urls.into_iter().enumerate().map(|(i, p)| {
+        let counter = Arc::clone(&counter);
+        let sem = Arc::clone(&sem);
+
+        async move {
+            let permit = sem.acquire_owned().await.unwrap();
+            let img = match get_post_images(&p).await {
+                Ok(img) => img,
+                Err(e) => {
+                    tracing::error!("Failed to retrieve info for {p} : {e}");
+                    return Err(e);
+                }
+            };
+            std::mem::drop(permit);
+            tracing::info!(
+                "Got info for {} of {total_posts}",
+                counter.fetch_add(1, Ordering::SeqCst)
+            );
+
+            Result::Ok((i, p, img))
+        }
+    }))
+    .await;

-    for (index, post) in post_urls.into_iter().enumerate() {
+    for result in post_results {
+        match result {
+            Ok(e) => {
+                posts_with_img.push(e);
+            }
+            Err(e) => {
+                tracing::error!("Failed to retrieve post info: {e}");
+            }
+        }
+    }
+
+    for (index, post, images) in posts_with_img {
        tracing::info!("Importing post {} of {}", index + 1, total_posts);
-        if let Err(e) = import_post(&post, hydrus).await {
-            tracing::error!("Failed to import {}: {}", post, e);
+        if let Err(e) = import_post(hydrus, &post, images).await {
+            tracing::error!("Failed to import post {}: {}", post, e);
        }
    }

    Ok(())
 }

-#[tracing::instrument(level = "debug", skip(hydrus))]
-async fn import_post(post_url: &str, hydrus: &Hydrus) -> Result<()> {
-    tracing::debug!("Post {}", post_url);
-    let images = get_post_images(post_url).await?;
-    tracing::info!("Found {} images for post {}", images.len(), post_url);
-
+async fn import_post(hydrus: &Hydrus, post: &String, images: Vec<String>) -> Result<()> {
    for url in images {
        let mut entry = hydrus.import().url(url).run().await?;
        let files = entry.files().await?;

        for mut file in files {
-            file.associate_urls(vec![post_url.to_string()]).await?;
+            file.associate_urls(vec![post.to_string()]).await?;
        }
    }
+
    Ok(())
 }
--- a/src/operations/find_and_send_tags.rs
+++ b/src/operations/find_and_send_tags.rs
@ -2,7 +2,7 @@ use crate::{
    error::Result,
    utils::pixiv::{get_pixiv_url, get_sauces_for_file, get_tags_for_sauce},
 };
-use hydrus_api::wrapper::{hydrus_file::HydrusFile, service::ServiceName};
+use hydrus_api::wrapper::hydrus_file::HydrusFile;
 use pixiv_rs::PixivClient;
 use rustnao::{Handler, Sauce};
 use tempdir::TempDir;
@ -12,15 +12,16 @@ pub async fn find_and_send_tags(
    finish_tag: Option<&String>,
    handler: &Handler,
    pixiv: &PixivClient,
-    service: &ServiceName,
+    service_key: &str,
    tmpdir: &TempDir,
    mut file: &mut HydrusFile,
 ) -> Result<()> {
-    if let Err(e) = search_and_assign_tags(&handler, &pixiv, &service, &tmpdir, &mut file).await {
+    if let Err(e) = search_and_assign_tags(&handler, &pixiv, service_key, &tmpdir, &mut file).await
+    {
        let hash = file.hash().await.unwrap();
        tracing::error!("Failed to search tags to file {}: {:?}", hash, e);
    } else if let Some(finish_tag) = finish_tag {
-        file.add_tags(service.clone().into(), vec![finish_tag.into()])
+        file.add_tags(service_key.to_owned(), vec![finish_tag.into()])
            .await
            .unwrap();
    }
@ -32,20 +33,20 @@ pub async fn find_and_send_tags(
 async fn search_and_assign_tags(
    handler: &Handler,
    pixiv: &PixivClient,
-    service: &ServiceName,
+    service_key: &str,
    tmpdir: &TempDir,
    mut file: &mut HydrusFile,
 ) -> Result<()> {
    tracing::debug!("Getting tags for hydrus file {:?}", file.id);
    let sauces = get_sauces_for_file(&handler, tmpdir, file).await?;

-    assign_pixiv_tags_and_url(&pixiv, service, &mut file, &sauces).await
+    assign_pixiv_tags_and_url(&pixiv, service_key, &mut file, &sauces).await
 }

 #[tracing::instrument(level = "debug", skip_all)]
 async fn assign_pixiv_tags_and_url(
    pixiv: &&PixivClient,
-    service: &ServiceName,
+    service_key: &str,
    file: &mut &mut HydrusFile,
    sauce: &Vec<Sauce>,
 ) -> Result<()> {
@ -55,7 +56,7 @@ async fn assign_pixiv_tags_and_url(

        if tags.len() > 0 {
            tracing::info!("Found {} tags for file {:?}", tags.len(), hash);
-            file.add_tags(service.clone().into(), tags).await?;
+            file.add_tags(service_key.to_owned(), tags).await?;
        } else {
            tracing::info!("No tags for file {:?} found", hash);
        }
--- a/src/operations/mod.rs
+++ b/src/operations/mod.rs
@ -1,3 +1,4 @@
+pub mod find_and_send_fedi_posts;
 pub mod find_and_send_reddit_posts;
 pub mod find_and_send_tags;
 pub mod find_and_send_urls;
--- a/src/utils/fedi.rs
+++ b/src/utils/fedi.rs
@ -0,0 +1,97 @@
+#![allow(unused)]
+use std::collections::HashMap;
+
+use crate::Result;
+use lazy_regex::regex;
+use reqwest::header::{HeaderMap, HeaderValue};
+use reqwest::ClientBuilder;
+use reqwest::{redirect::Policy, StatusCode};
+use serde::Deserialize;
+use serde_json::Value;
+use std::fmt::Debug;
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type")]
+enum EntryData {
+    Page(PostData),
+}
+
+#[derive(Debug, Deserialize)]
+struct PostData {
+    id: String,
+    name: String,
+    attachment: Vec<Attachment>,
+    #[serde(flatten)]
+    _extra: HashMap<String, Value>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type")]
+enum Attachment {
+    Link { href: String },
+}
+
+pub async fn is_fedi_url(url: &str) -> bool {
+    get_post(url).await.is_ok()
+}
+
+/// Returns all images associated with a post
+#[tracing::instrument(level = "debug")]
+pub async fn get_post_images<S: AsRef<str> + Debug>(post_url: S) -> Result<Vec<String>> {
+    let post_data = get_post(post_url.as_ref()).await?;
+
+    let urls = post_data
+        .attachment
+        .into_iter()
+        .map(|p| {
+            let Attachment::Link { href } = p;
+            href
+        })
+        .collect();
+
+    Ok(urls)
+}
+
+#[tracing::instrument(level = "debug")]
+async fn get_post(url: &str) -> Result<PostData> {
+    let mut headers = HeaderMap::new();
+    headers.insert(
+        "Accept",
+        HeaderValue::from_static("application/activity+json"),
+    );
+
+    let client = ClientBuilder::default()
+        .default_headers(headers)
+        .user_agent(fakeit::user_agent::random_platform())
+        .build()?;
+    let mut response: EntryData = client.get(url).send().await?.json().await?;
+
+    let EntryData::Page(post) = response;
+
+    Ok(post)
+}
+
+#[tokio::test]
+async fn it_retrieves_post_data() {
+    let data = get_post("https://lemmy.blahaj.zone/post/113727")
+        .await
+        .unwrap();
+    assert!(!data.attachment.is_empty());
+}
+
+#[tokio::test]
+async fn it_retrieves_post_misskey() {
+    let data = get_post("https://social.funkyfish.cool/notes/97ng0c9is3")
+        .await
+        .unwrap();
+    assert!(!data.attachment.is_empty());
+}
+
+#[tokio::test]
+async fn it_retrieves_post_images() {
+    let images = get_post_images("https://lemmy.blahaj.zone/post/113727")
+        .await
+        .unwrap();
+    assert!(!images.is_empty());
+    assert!(images.get(0).unwrap().ends_with(".jpg"));
+}
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@ -1,6 +1,8 @@
+pub mod fedi;
 pub mod pixiv;
 pub mod reddit;
 pub mod urls;
+
 use crate::error::Result;
 use directories::ProjectDirs;
 use std::{fs, path::PathBuf};
--- a/src/utils/reddit.rs
+++ b/src/utils/reddit.rs
@ -111,22 +111,29 @@ async fn get_post(url: &str) -> Result<T3Data> {
 /// Resolves reddit redirects
 #[tracing::instrument(level = "debug")]
 async fn resolve_redirects(url: &str) -> Result<String> {
-    if is_resolved(url) {
-        tracing::debug!("Url already resolved.");
-        return Ok(url.to_string());
+    let mut url = url.to_string();
+
+    for _ in 0..10 {
+        if is_resolved(&url) {
+            tracing::debug!("Url already resolved.");
+            return Ok(url);
+        }
+        let client = reqwest::Client::builder()
+            .user_agent(fakeit::user_agent::random_platform())
+            .redirect(Policy::none())
+            .build()?;
+        let response = client.get(url).send().await?;
+
+        if let Some(location) = response.headers().get("location") {
+            tracing::debug!("Redirect to {location:?} found");
+            url = location.to_str().unwrap().to_string();
+        } else {
+            tracing::debug!("No redirect found.");
+            return Ok(response.url().as_str().to_string());
+        }
    }
-    let client = reqwest::Client::builder()
-        .redirect(Policy::none())
-        .build()?;
-    let response = client.head(url).send().await?;

-    if let Some(location) = response.headers().get("location") {
-        tracing::debug!("Redirect to {location:?} found");
-        Ok(location.to_str().unwrap().to_string())
-    } else {
-        tracing::debug!("No redirect found.");
-        Ok(response.url().as_str().to_string())
-    }
+    Ok(url)
 }

 /// Checks if the url is already in a format that can be used for retrieving information
@ -147,6 +154,14 @@ mod test {
        assert!(images.is_empty() == false);
    }

+    #[tokio::test]
+    async fn it_finds_post_images2() {
+        let images = super::get_post_images("https://reddit.com/r/HentaiBullying/s/S1gKoG4s2S/")
+            .await
+            .unwrap();
+        assert!(images.is_empty() == false);
+    }
+
    #[tokio::test]
    async fn it_finds_multiple_post_images() {
        let images =
@ -164,7 +179,6 @@ mod test {
        println!("{:?}", post.url);
        assert!(post.url.is_some());
    }
-
    #[tokio::test]
    async fn it_finds_info_for_gallery_posts() {
        let post = super::get_post("https://www.reddit.com/r/dogelore/comments/wmas8c/le_yakuza/")
--- a/src/utils/urls.rs
+++ b/src/utils/urls.rs
@ -1,13 +1,17 @@
+use super::fedi;
 use lazy_regex::regex;

 pub enum UrlType {
    Reddit,
+    Fedi,
    Other,
 }

-pub fn find_url_type(url: &str) -> UrlType {
+pub async fn find_url_type(url: &str) -> UrlType {
    if is_reddit_url(url) {
        UrlType::Reddit
+    } else if fedi::is_fedi_url(url).await {
+        UrlType::Fedi
    } else {
        UrlType::Other
    }
Author	SHA1	Message	Date
trivernis	d31811e3b7	chore: Release hydrus-utils version 0.6.0	5 months ago
trivernis	6c73686387	Add tags command to add tags to files and update hydrus api	5 months ago
trivernis	dbc5275f5d	Fix reddit wanting a user agent now	8 months ago
trivernis	f8228a5856	Improve reddit import speed	11 months ago
trivernis	80fd7488c7	Add support for fedi posts (lemmy in particular)	11 months ago
trivernis	5c2339a8d9	Fix rate limit calculation issue	11 months ago