Improve thunmbnail generation

Signed-off-by: trivernis <trivernis@protonmail.com>

mediarepo-daemon/Cargo.lock

@@ -829,6 +829,7 @@ dependencies = [
  "mediarepo-core",
  "mediarepo-model",
  "mediarepo-socket",
+ "num-integer",
  "structopt",
  "tokio",
  "toml",

mediarepo-daemon/Cargo.toml

@@ -20,7 +20,7 @@ glob = "0.3.0"
 log = "0.4.14"
 tracing-flame = "0.1.0"
 tracing-appender = "0.2.0"
-
+num-integer = "0.1.44"
 
 [dependencies.mediarepo-core]
 path = "./mediarepo-core"

mediarepo-daemon/mediarepo-model/src/repo.rs

@@ -100,10 +100,14 @@ impl Repo {
     /// Finds all files by a list of tags
     #[tracing::instrument(level = "debug", skip(self))]
     pub async fn find_files_by_tags(&self, tags: Vec<(String, bool)>) -> RepoResult<Vec<File>> {
-        let db_tags = self
+        let parsed_tags = tags
-            .find_all_tags(tags.iter().map(|t| t.0.clone()).collect())
+            .iter()
-            .await?;
+            .map(|t| parse_namespace_and_tag(t.0.clone()))
+            .collect();
+
+        let db_tags = self.find_all_tags(parsed_tags).await?;
         let tag_map: HashMap<String, bool> = HashMap::from_iter(tags.into_iter());
+
         let tag_ids: Vec<(i64, bool)> = db_tags
             .into_iter()
             .map(|tag| {
@@ -188,9 +192,7 @@ impl Repo {
 
     /// Finds all tags by name
     #[tracing::instrument(level = "debug", skip(self))]
-    pub async fn find_all_tags(&self, tags: Vec<String>) -> RepoResult<Vec<Tag>> {
+    pub async fn find_all_tags(&self, tags: Vec<(Option<String>, String)>) -> RepoResult<Vec<Tag>> {
-        let tags: Vec<(Option<String>, String)> =
-            tags.into_iter().map(parse_namespace_and_tag).collect();
         Tag::all_by_name(self.db.clone(), tags).await
     }
 
@@ -217,7 +219,7 @@ impl Repo {
 
     /// Adds an unnamespaced tag
     #[tracing::instrument(level = "debug", skip(self))]
-    async fn add_unnamespaced_tag(&self, name: String) -> RepoResult<Tag> {
+    pub async fn add_unnamespaced_tag(&self, name: String) -> RepoResult<Tag> {
         Tag::add(self.db.clone(), name, None).await
     }
 
@@ -237,7 +239,7 @@ impl Repo {
 
     /// Adds a namespaced tag
     #[tracing::instrument(level = "debug", skip(self))]
-    async fn add_namespaced_tag(&self, name: String, namespace: String) -> RepoResult<Tag> {
+    pub async fn add_namespaced_tag(&self, name: String, namespace: String) -> RepoResult<Tag> {
         let namespace =
             if let Some(namespace) = Namespace::by_name(self.db.clone(), &namespace).await? {
                 namespace

mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs

@@ -107,7 +107,14 @@ impl FilesNamespace {
         let request = event.data::<GetFileThumbnailsRequest>()?;
         let repo = get_repo_from_context(ctx).await;
         let file = file_by_identifier(request.id, &repo).await?;
-        let thumbnails = file.thumbnails().await?;
+        let mut thumbnails = file.thumbnails().await?;
+
+        if thumbnails.len() == 0 {
+            tracing::debug!("No thumbnails for file found. Creating thumbnails...");
+            repo.create_thumbnails_for_file(file.clone()).await?;
+            tracing::debug!("Thumbnails for file created.");
+        }
+        thumbnails = file.thumbnails().await?;
 
         let thumb_responses: Vec<ThumbnailMetadataResponse> = thumbnails
             .into_iter()

mediarepo-daemon/src/main.rs

@@ -7,13 +7,15 @@ use tokio::runtime;
 use tokio::runtime::Runtime;
 
 use mediarepo_core::error::RepoResult;
+use mediarepo_core::futures;
 use mediarepo_core::settings::Settings;
 use mediarepo_core::type_keys::SettingsKey;
 use mediarepo_core::utils::parse_tags_file;
-use mediarepo_model::file::File as RepoFile;
+use mediarepo_model::file::{File as RepoFile, File};
 use mediarepo_model::repo::Repo;
 use mediarepo_model::type_keys::RepoKey;
 use mediarepo_socket::get_builder;
+use num_integer::Integer;
 
 use crate::constants::{DEFAULT_STORAGE_NAME, SETTINGS_PATH, THUMBNAIL_STORAGE_NAME};
 use crate::utils::{create_paths_for_repo, get_repo, load_settings};
@@ -52,6 +54,10 @@ enum SubCommand {
         /// The path to the folder where the files are located
         #[structopt()]
         folder_path: String,
+
+        /// If imported files should be deleted after import
+        #[structopt(long)]
+        delete: bool,
     },
 
     /// Starts the event server for the selected repository
@@ -70,9 +76,10 @@ fn main() -> RepoResult<()> {
     match opt.cmd.clone() {
         SubCommand::Init { force } => get_single_thread_runtime().block_on(init(opt, force)),
         SubCommand::Start => get_multi_thread_runtime().block_on(start_server(opt)),
-        SubCommand::Import { folder_path } => {
+        SubCommand::Import {
-            get_single_thread_runtime().block_on(import(opt, folder_path))
+            folder_path,
-        }
+            delete,
+        } => get_single_thread_runtime().block_on(import(opt, folder_path, delete)),
     }?;
 
     Ok(())
@@ -152,7 +159,7 @@ async fn init(opt: Opt, force: bool) -> RepoResult<()> {
 }
 
 /// Imports files from a source into the database
-async fn import(opt: Opt, path: String) -> RepoResult<()> {
+async fn import(opt: Opt, path: String, delete_files: bool) -> RepoResult<()> {
     let (_s, repo) = init_repo(&opt).await?;
     log::info!("Importing");
 
@@ -164,20 +171,42 @@ async fn import(opt: Opt, path: String) -> RepoResult<()> {
         .collect();
 
     for path in paths {
-        if let Err(e) = import_single_image(path, &repo).await {
+        if let Err(e) = import_single_image(&path, &repo).await {
             log::error!("Import failed: {:?}", e);
+            if delete_files {
+                log::info!("Deleting file {:?}", path);
+                let _ = fs::remove_file(&path).await;
+            }
+        } else {
+            if delete_files {
+                log::info!("Deleting file {:?}", path);
+                let _ = fs::remove_file(&path).await;
+            }
         }
     }
+    log::info!("Creating thumbnails...");
+    let mut files = repo.files().await?;
+
+    for _ in 0..(files.len().div_ceil(&64)) {
+        futures::future::join_all(
+            (0..64)
+                .filter_map(|_| files.pop())
+                .map(|f| create_file_thumbnails(&repo, f)),
+        )
+        .await
+        .into_iter()
+        .filter_map(|r| r.err())
+        .for_each(|e| log::error!("Failed to create thumbnail: {:?}", e));
+    }
 
     Ok(())
 }
 
 /// Creates thumbnails of all sizes
-async fn import_single_image(path: PathBuf, repo: &Repo) -> RepoResult<()> {
+async fn import_single_image(path: &PathBuf, repo: &Repo) -> RepoResult<()> {
     log::info!("Importing file");
     let file = repo.add_file_by_path(path.clone()).await?;
-    log::info!("Creating thumbnails");
+    log::info!("Adding tags");
-    repo.create_thumbnails_for_file(file.clone()).await?;
     let tags_path = PathBuf::from(format!("{}{}", path.to_str().unwrap(), ".txt"));
     add_tags_from_tags_file(tags_path, repo, file).await?;
 
@@ -191,16 +220,25 @@ async fn add_tags_from_tags_file(
 ) -> RepoResult<()> {
     log::info!("Adding tags");
     if tags_path.exists() {
-        let tags = parse_tags_file(tags_path).await?;
+        let mut tags = parse_tags_file(tags_path).await?;
-        let mut tag_ids = Vec::new();
+        let resolved_tags = repo.find_all_tags(tags.clone()).await?;
+        tags.retain(|tag| {
+            resolved_tags
+                .iter()
+                .find(|t| if let (Some(ns1), Some(ns2)) = (t.namespace(), &tag.0) {
+                    *ns1.name() == *ns2
+                } else { false } && *t.name() == *tag.1)
+                .is_some()
+        });
+        let mut tag_ids: Vec<i64> = resolved_tags.into_iter().map(|t| t.id()).collect();
 
         for (namespace, name) in tags {
             let tag = if let Some(namespace) = namespace {
                 log::info!("Adding namespaced tag '{}:{}'", namespace, name);
-                repo.add_or_find_namespaced_tag(name, namespace).await?
+                repo.add_namespaced_tag(name, namespace).await?
             } else {
                 log::info!("Adding unnamespaced tag '{}'", name);
-                repo.add_or_find_unnamespaced_tag(name).await?
+                repo.add_unnamespaced_tag(name).await?
             };
             tag_ids.push(tag.id());
         }
@@ -211,3 +249,11 @@ async fn add_tags_from_tags_file(
     }
     Ok(())
 }
+
+#[tracing::instrument(skip(repo, file))]
+async fn create_file_thumbnails(repo: &Repo, file: File) -> RepoResult<()> {
+    if file.thumbnails().await?.len() == 0 {
+        repo.create_thumbnails_for_file(file).await?;
+    }
+    Ok(())
+}