Improve importing of reddit posts

Signed-off-by: trivernis <trivernis@protonmail.com>
main
trivernis 2 years ago
parent 7e1e1d9de1
commit 5a7affcf4a
Signed by: Trivernis
GPG Key ID: DFFFCC2C7A02DB45

1
.gitignore vendored

@ -2,3 +2,4 @@
reddit-urls.txt
reddt-urls.txt
urls.txt
.hydrus-utils.toml

@ -37,9 +37,14 @@ impl Config {
if !config_file_path.exists() {
fs::write(&config_file_path, include_str!("assets/config.toml"))?;
}
let settings = config::Config::builder()
.add_source(config::File::with_name(config_file_path.to_str().unwrap()))
.build()?;
let mut builder = config::Config::builder()
.add_source(config::File::with_name(config_file_path.to_str().unwrap()));
let local_config = PathBuf::from(".hydrus-utils.toml");
if local_config.exists() {
builder = builder.add_source(config::File::with_name(".hydrus-utils.toml"));
}
let settings = builder.build()?;
tracing::debug!("Config is {settings:?}");
Ok(settings.try_deserialize()?)

@ -83,9 +83,12 @@ async fn send_tags_or_urls(
let tmpdir = TempDir::new("hydrus-files").unwrap();
let sleep_duration = Duration::from_secs(6);
let total_files = files.len();
for mut file in files {
for (i, mut file) in files.into_iter().enumerate() {
let start = Instant::now();
tracing::info!("Searching for file {} out of {}", i + 1, total_files);
if send_urls {
let _ = find_and_send_urls(&hydrus, &handler, &tmpdir, &mut file).await;
} else {

@ -9,14 +9,22 @@ pub async fn find_and_send_reddit_posts(hydrus: &Hydrus, post_urls: Vec<String>)
for (index, post) in post_urls.into_iter().enumerate() {
tracing::info!("Importing post {} of {}", index + 1, total_posts);
tracing::debug!("Post {}", post);
let images = get_post_images(&post).await?;
tracing::info!("Found {} images for post {}", images.len(), post);
for url in images {
hydrus.import().url(url).run().await?;
if let Err(e) = import_post(&post, hydrus).await {
tracing::error!("Failed to import {}: {}", post, e);
}
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(hydrus))]
async fn import_post(post_url: &str, hydrus: &Hydrus) -> Result<()> {
tracing::debug!("Post {}", post_url);
let images = get_post_images(post_url).await?;
tracing::info!("Found {} images for post {}", images.len(), post_url);
for url in images {
hydrus.import().url(url).run().await?;
}
Ok(())
}

@ -7,6 +7,7 @@ use pixiv_rs::PixivClient;
use rustnao::{Handler, Sauce};
use tempdir::TempDir;
#[tracing::instrument(level = "debug", skip_all)]
pub async fn find_and_send_tags(
finish_tag: Option<&String>,
handler: &Handler,
@ -27,6 +28,7 @@ pub async fn find_and_send_tags(
Ok(())
}
#[tracing::instrument(level = "debug", skip_all)]
async fn search_and_assign_tags(
handler: &Handler,
pixiv: &PixivClient,
@ -40,6 +42,7 @@ async fn search_and_assign_tags(
assign_pixiv_tags_and_url(&pixiv, service, &mut file, &sauces).await
}
#[tracing::instrument(level = "debug", skip_all)]
async fn assign_pixiv_tags_and_url(
pixiv: &&PixivClient,
service: &ServiceName,

@ -5,6 +5,7 @@ use tempdir::TempDir;
use crate::error::Result;
use crate::utils::pixiv::{get_sauces_for_file, get_urls};
#[tracing::instrument(level = "debug", skip_all)]
pub async fn find_and_send_urls(
hydrus: &Hydrus,
handler: &Handler,

Loading…
Cancel
Save