diff --git a/mediarepo-daemon/Cargo.lock b/mediarepo-daemon/Cargo.lock index dec02f0..9f2f1c3 100644 --- a/mediarepo-daemon/Cargo.lock +++ b/mediarepo-daemon/Cargo.lock @@ -829,6 +829,7 @@ dependencies = [ "mediarepo-core", "mediarepo-model", "mediarepo-socket", + "num-integer", "structopt", "tokio", "toml", diff --git a/mediarepo-daemon/Cargo.toml b/mediarepo-daemon/Cargo.toml index 3f4848e..6eb7984 100644 --- a/mediarepo-daemon/Cargo.toml +++ b/mediarepo-daemon/Cargo.toml @@ -20,7 +20,7 @@ glob = "0.3.0" log = "0.4.14" tracing-flame = "0.1.0" tracing-appender = "0.2.0" - +num-integer = "0.1.44" [dependencies.mediarepo-core] path = "./mediarepo-core" diff --git a/mediarepo-daemon/mediarepo-model/src/repo.rs b/mediarepo-daemon/mediarepo-model/src/repo.rs index 1fe56cf..62acb42 100644 --- a/mediarepo-daemon/mediarepo-model/src/repo.rs +++ b/mediarepo-daemon/mediarepo-model/src/repo.rs @@ -100,10 +100,14 @@ impl Repo { /// Finds all files by a list of tags #[tracing::instrument(level = "debug", skip(self))] pub async fn find_files_by_tags(&self, tags: Vec<(String, bool)>) -> RepoResult> { - let db_tags = self - .find_all_tags(tags.iter().map(|t| t.0.clone()).collect()) - .await?; + let parsed_tags = tags + .iter() + .map(|t| parse_namespace_and_tag(t.0.clone())) + .collect(); + + let db_tags = self.find_all_tags(parsed_tags).await?; let tag_map: HashMap = HashMap::from_iter(tags.into_iter()); + let tag_ids: Vec<(i64, bool)> = db_tags .into_iter() .map(|tag| { @@ -188,9 +192,7 @@ impl Repo { /// Finds all tags by name #[tracing::instrument(level = "debug", skip(self))] - pub async fn find_all_tags(&self, tags: Vec) -> RepoResult> { - let tags: Vec<(Option, String)> = - tags.into_iter().map(parse_namespace_and_tag).collect(); + pub async fn find_all_tags(&self, tags: Vec<(Option, String)>) -> RepoResult> { Tag::all_by_name(self.db.clone(), tags).await } @@ -217,7 +219,7 @@ impl Repo { /// Adds an unnamespaced tag #[tracing::instrument(level = "debug", skip(self))] - async fn add_unnamespaced_tag(&self, name: String) -> RepoResult { + pub async fn add_unnamespaced_tag(&self, name: String) -> RepoResult { Tag::add(self.db.clone(), name, None).await } @@ -237,7 +239,7 @@ impl Repo { /// Adds a namespaced tag #[tracing::instrument(level = "debug", skip(self))] - async fn add_namespaced_tag(&self, name: String, namespace: String) -> RepoResult { + pub async fn add_namespaced_tag(&self, name: String, namespace: String) -> RepoResult { let namespace = if let Some(namespace) = Namespace::by_name(self.db.clone(), &namespace).await? { namespace diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs index c36dea2..c2ccb6e 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs @@ -107,7 +107,14 @@ impl FilesNamespace { let request = event.data::()?; let repo = get_repo_from_context(ctx).await; let file = file_by_identifier(request.id, &repo).await?; - let thumbnails = file.thumbnails().await?; + let mut thumbnails = file.thumbnails().await?; + + if thumbnails.len() == 0 { + tracing::debug!("No thumbnails for file found. Creating thumbnails..."); + repo.create_thumbnails_for_file(file.clone()).await?; + tracing::debug!("Thumbnails for file created."); + } + thumbnails = file.thumbnails().await?; let thumb_responses: Vec = thumbnails .into_iter() diff --git a/mediarepo-daemon/src/main.rs b/mediarepo-daemon/src/main.rs index 403eb4d..a42a52c 100644 --- a/mediarepo-daemon/src/main.rs +++ b/mediarepo-daemon/src/main.rs @@ -7,13 +7,15 @@ use tokio::runtime; use tokio::runtime::Runtime; use mediarepo_core::error::RepoResult; +use mediarepo_core::futures; use mediarepo_core::settings::Settings; use mediarepo_core::type_keys::SettingsKey; use mediarepo_core::utils::parse_tags_file; -use mediarepo_model::file::File as RepoFile; +use mediarepo_model::file::{File as RepoFile, File}; use mediarepo_model::repo::Repo; use mediarepo_model::type_keys::RepoKey; use mediarepo_socket::get_builder; +use num_integer::Integer; use crate::constants::{DEFAULT_STORAGE_NAME, SETTINGS_PATH, THUMBNAIL_STORAGE_NAME}; use crate::utils::{create_paths_for_repo, get_repo, load_settings}; @@ -52,6 +54,10 @@ enum SubCommand { /// The path to the folder where the files are located #[structopt()] folder_path: String, + + /// If imported files should be deleted after import + #[structopt(long)] + delete: bool, }, /// Starts the event server for the selected repository @@ -70,9 +76,10 @@ fn main() -> RepoResult<()> { match opt.cmd.clone() { SubCommand::Init { force } => get_single_thread_runtime().block_on(init(opt, force)), SubCommand::Start => get_multi_thread_runtime().block_on(start_server(opt)), - SubCommand::Import { folder_path } => { - get_single_thread_runtime().block_on(import(opt, folder_path)) - } + SubCommand::Import { + folder_path, + delete, + } => get_single_thread_runtime().block_on(import(opt, folder_path, delete)), }?; Ok(()) @@ -152,7 +159,7 @@ async fn init(opt: Opt, force: bool) -> RepoResult<()> { } /// Imports files from a source into the database -async fn import(opt: Opt, path: String) -> RepoResult<()> { +async fn import(opt: Opt, path: String, delete_files: bool) -> RepoResult<()> { let (_s, repo) = init_repo(&opt).await?; log::info!("Importing"); @@ -164,20 +171,42 @@ async fn import(opt: Opt, path: String) -> RepoResult<()> { .collect(); for path in paths { - if let Err(e) = import_single_image(path, &repo).await { + if let Err(e) = import_single_image(&path, &repo).await { log::error!("Import failed: {:?}", e); + if delete_files { + log::info!("Deleting file {:?}", path); + let _ = fs::remove_file(&path).await; + } + } else { + if delete_files { + log::info!("Deleting file {:?}", path); + let _ = fs::remove_file(&path).await; + } } } + log::info!("Creating thumbnails..."); + let mut files = repo.files().await?; + + for _ in 0..(files.len().div_ceil(&64)) { + futures::future::join_all( + (0..64) + .filter_map(|_| files.pop()) + .map(|f| create_file_thumbnails(&repo, f)), + ) + .await + .into_iter() + .filter_map(|r| r.err()) + .for_each(|e| log::error!("Failed to create thumbnail: {:?}", e)); + } Ok(()) } /// Creates thumbnails of all sizes -async fn import_single_image(path: PathBuf, repo: &Repo) -> RepoResult<()> { +async fn import_single_image(path: &PathBuf, repo: &Repo) -> RepoResult<()> { log::info!("Importing file"); let file = repo.add_file_by_path(path.clone()).await?; - log::info!("Creating thumbnails"); - repo.create_thumbnails_for_file(file.clone()).await?; + log::info!("Adding tags"); let tags_path = PathBuf::from(format!("{}{}", path.to_str().unwrap(), ".txt")); add_tags_from_tags_file(tags_path, repo, file).await?; @@ -191,16 +220,25 @@ async fn add_tags_from_tags_file( ) -> RepoResult<()> { log::info!("Adding tags"); if tags_path.exists() { - let tags = parse_tags_file(tags_path).await?; - let mut tag_ids = Vec::new(); + let mut tags = parse_tags_file(tags_path).await?; + let resolved_tags = repo.find_all_tags(tags.clone()).await?; + tags.retain(|tag| { + resolved_tags + .iter() + .find(|t| if let (Some(ns1), Some(ns2)) = (t.namespace(), &tag.0) { + *ns1.name() == *ns2 + } else { false } && *t.name() == *tag.1) + .is_some() + }); + let mut tag_ids: Vec = resolved_tags.into_iter().map(|t| t.id()).collect(); for (namespace, name) in tags { let tag = if let Some(namespace) = namespace { log::info!("Adding namespaced tag '{}:{}'", namespace, name); - repo.add_or_find_namespaced_tag(name, namespace).await? + repo.add_namespaced_tag(name, namespace).await? } else { log::info!("Adding unnamespaced tag '{}'", name); - repo.add_or_find_unnamespaced_tag(name).await? + repo.add_unnamespaced_tag(name).await? }; tag_ids.push(tag.id()); } @@ -211,3 +249,11 @@ async fn add_tags_from_tags_file( } Ok(()) } + +#[tracing::instrument(skip(repo, file))] +async fn create_file_thumbnails(repo: &Repo, file: File) -> RepoResult<()> { + if file.thumbnails().await?.len() == 0 { + repo.create_thumbnails_for_file(file).await?; + } + Ok(()) +}