From 5a7affcf4a4c11bad74e84e826710864a4029b11 Mon Sep 17 00:00:00 2001
From: trivernis <trivernis@protonmail.com>
Date: Tue, 23 Aug 2022 15:56:09 +0200
Subject: [PATCH] Improve importing of reddit posts

Signed-off-by: trivernis <trivernis@protonmail.com>
---
 .gitignore                                   |  1 +
 src/config.rs                                | 11 ++++++++---
 src/main.rs                                  |  5 ++++-
 src/operations/find_and_send_reddit_posts.rs | 20 ++++++++++++++------
 src/operations/find_and_send_tags.rs         |  3 +++
 src/operations/find_and_send_urls.rs         |  1 +
 6 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/.gitignore b/.gitignore
index 3833b7e..f8e518a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 reddit-urls.txt
 reddt-urls.txt
 urls.txt
+.hydrus-utils.toml
diff --git a/src/config.rs b/src/config.rs
index 19ef58a..41e58c6 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -37,9 +37,14 @@ impl Config {
         if !config_file_path.exists() {
             fs::write(&config_file_path, include_str!("assets/config.toml"))?;
         }
-        let settings = config::Config::builder()
-            .add_source(config::File::with_name(config_file_path.to_str().unwrap()))
-            .build()?;
+        let mut builder = config::Config::builder()
+            .add_source(config::File::with_name(config_file_path.to_str().unwrap()));
+
+        let local_config = PathBuf::from(".hydrus-utils.toml");
+        if local_config.exists() {
+            builder = builder.add_source(config::File::with_name(".hydrus-utils.toml"));
+        }
+        let settings = builder.build()?;
         tracing::debug!("Config is {settings:?}");
 
         Ok(settings.try_deserialize()?)
diff --git a/src/main.rs b/src/main.rs
index fc62bfe..d9eae9b 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -83,9 +83,12 @@ async fn send_tags_or_urls(
     let tmpdir = TempDir::new("hydrus-files").unwrap();
 
     let sleep_duration = Duration::from_secs(6);
+    let total_files = files.len();
 
-    for mut file in files {
+    for (i, mut file) in files.into_iter().enumerate() {
         let start = Instant::now();
+        tracing::info!("Searching for file {} out of {}", i + 1, total_files);
+
         if send_urls {
             let _ = find_and_send_urls(&hydrus, &handler, &tmpdir, &mut file).await;
         } else {
diff --git a/src/operations/find_and_send_reddit_posts.rs b/src/operations/find_and_send_reddit_posts.rs
index d862f7c..314444e 100644
--- a/src/operations/find_and_send_reddit_posts.rs
+++ b/src/operations/find_and_send_reddit_posts.rs
@@ -9,14 +9,22 @@ pub async fn find_and_send_reddit_posts(hydrus: &Hydrus, post_urls: Vec<String>)
 
     for (index, post) in post_urls.into_iter().enumerate() {
         tracing::info!("Importing post {} of {}", index + 1, total_posts);
-        tracing::debug!("Post {}", post);
-        let images = get_post_images(&post).await?;
-        tracing::info!("Found {} images for post {}", images.len(), post);
-
-        for url in images {
-            hydrus.import().url(url).run().await?;
+        if let Err(e) = import_post(&post, hydrus).await {
+            tracing::error!("Failed to import {}: {}", post, e);
         }
     }
 
     Ok(())
 }
+
+#[tracing::instrument(level = "debug", skip(hydrus))]
+async fn import_post(post_url: &str, hydrus: &Hydrus) -> Result<()> {
+    tracing::debug!("Post {}", post_url);
+    let images = get_post_images(post_url).await?;
+    tracing::info!("Found {} images for post {}", images.len(), post_url);
+
+    for url in images {
+        hydrus.import().url(url).run().await?;
+    }
+    Ok(())
+}
diff --git a/src/operations/find_and_send_tags.rs b/src/operations/find_and_send_tags.rs
index 7d621de..e0e11f0 100644
--- a/src/operations/find_and_send_tags.rs
+++ b/src/operations/find_and_send_tags.rs
@@ -7,6 +7,7 @@ use pixiv_rs::PixivClient;
 use rustnao::{Handler, Sauce};
 use tempdir::TempDir;
 
+#[tracing::instrument(level = "debug", skip_all)]
 pub async fn find_and_send_tags(
     finish_tag: Option<&String>,
     handler: &Handler,
@@ -27,6 +28,7 @@ pub async fn find_and_send_tags(
     Ok(())
 }
 
+#[tracing::instrument(level = "debug", skip_all)]
 async fn search_and_assign_tags(
     handler: &Handler,
     pixiv: &PixivClient,
@@ -40,6 +42,7 @@ async fn search_and_assign_tags(
     assign_pixiv_tags_and_url(&pixiv, service, &mut file, &sauces).await
 }
 
+#[tracing::instrument(level = "debug", skip_all)]
 async fn assign_pixiv_tags_and_url(
     pixiv: &&PixivClient,
     service: &ServiceName,
diff --git a/src/operations/find_and_send_urls.rs b/src/operations/find_and_send_urls.rs
index 9a36843..8dfd6ea 100644
--- a/src/operations/find_and_send_urls.rs
+++ b/src/operations/find_and_send_urls.rs
@@ -5,6 +5,7 @@ use tempdir::TempDir;
 use crate::error::Result;
 use crate::utils::pixiv::{get_sauces_for_file, get_urls};
 
+#[tracing::instrument(level = "debug", skip_all)]
 pub async fn find_and_send_urls(
     hydrus: &Hydrus,
     handler: &Handler,