Add support for fedi posts (lemmy in particular)
parent
5c2339a8d9
commit
80fd7488c7
@ -0,0 +1,35 @@
|
|||||||
|
use hydrus_api::Hydrus;
|
||||||
|
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::utils::fedi::get_post_images;
|
||||||
|
|
||||||
|
#[tracing::instrument(level = "debug", skip(hydrus))]
|
||||||
|
pub async fn find_and_send_fedi_posts(hydrus: &Hydrus, post_urls: Vec<String>) -> Result<()> {
|
||||||
|
let total_posts = post_urls.len();
|
||||||
|
|
||||||
|
for (index, post) in post_urls.into_iter().enumerate() {
|
||||||
|
tracing::info!("Importing post {} of {}", index + 1, total_posts);
|
||||||
|
if let Err(e) = import_post(&post, hydrus).await {
|
||||||
|
tracing::error!("Failed to import {}: {}", post, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tracing::instrument(level = "debug", skip(hydrus))]
|
||||||
|
async fn import_post(post_url: &str, hydrus: &Hydrus) -> Result<()> {
|
||||||
|
tracing::debug!("Post {}", post_url);
|
||||||
|
let images = get_post_images(post_url).await?;
|
||||||
|
tracing::info!("Found {} images for post {}", images.len(), post_url);
|
||||||
|
|
||||||
|
for url in images {
|
||||||
|
let mut entry = hydrus.import().url(url).run().await?;
|
||||||
|
let files = entry.files().await?;
|
||||||
|
|
||||||
|
for mut file in files {
|
||||||
|
file.associate_urls(vec![post_url.to_string()]).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
@ -1,3 +1,4 @@
|
|||||||
|
pub mod find_and_send_fedi_posts;
|
||||||
pub mod find_and_send_reddit_posts;
|
pub mod find_and_send_reddit_posts;
|
||||||
pub mod find_and_send_tags;
|
pub mod find_and_send_tags;
|
||||||
pub mod find_and_send_urls;
|
pub mod find_and_send_urls;
|
||||||
|
@ -0,0 +1,89 @@
|
|||||||
|
#![allow(unused)]
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use crate::Result;
|
||||||
|
use lazy_regex::regex;
|
||||||
|
use reqwest::header::{HeaderMap, HeaderValue};
|
||||||
|
use reqwest::ClientBuilder;
|
||||||
|
use reqwest::{redirect::Policy, StatusCode};
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde_json::Value;
|
||||||
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(tag = "type")]
|
||||||
|
enum EntryData {
|
||||||
|
Page(PostData),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct PostData {
|
||||||
|
id: String,
|
||||||
|
name: String,
|
||||||
|
attachment: Vec<Attachment>,
|
||||||
|
#[serde(flatten)]
|
||||||
|
_extra: HashMap<String, Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(tag = "type")]
|
||||||
|
enum Attachment {
|
||||||
|
Link { href: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn is_fedi_url(url: &str) -> bool {
|
||||||
|
get_post(url).await.is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns all images associated with a post
|
||||||
|
#[tracing::instrument(level = "debug")]
|
||||||
|
pub async fn get_post_images<S: AsRef<str> + Debug>(post_url: S) -> Result<Vec<String>> {
|
||||||
|
let post_data = get_post(post_url.as_ref()).await?;
|
||||||
|
|
||||||
|
let urls = post_data
|
||||||
|
.attachment
|
||||||
|
.into_iter()
|
||||||
|
.map(|p| {
|
||||||
|
let Attachment::Link { href } = p;
|
||||||
|
href
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(urls)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tracing::instrument(level = "debug")]
|
||||||
|
async fn get_post(url: &str) -> Result<PostData> {
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert(
|
||||||
|
"Accept",
|
||||||
|
HeaderValue::from_static("application/activity+json"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let client = ClientBuilder::default()
|
||||||
|
.default_headers(headers)
|
||||||
|
.user_agent(fakeit::user_agent::random_platform())
|
||||||
|
.build()?;
|
||||||
|
let mut response: EntryData = client.get(url).send().await?.json().await?;
|
||||||
|
|
||||||
|
let EntryData::Page(post) = response;
|
||||||
|
|
||||||
|
Ok(post)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn it_retrieves_post_data() {
|
||||||
|
let data = get_post("https://lemmy.blahaj.zone/post/113727")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!data.attachment.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn it_retrieves_post_images() {
|
||||||
|
let images = get_post_images("https://lemmy.blahaj.zone/post/113727")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!images.is_empty());
|
||||||
|
assert!(images.get(0).unwrap().ends_with(".jpg"));
|
||||||
|
}
|
Loading…
Reference in New Issue